From 5dc8f27f9e4960d7d42f60b3eb4af707afcbe1c8 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 09:30:20 +0000 Subject: [PATCH 01/67] docs(plans): add resumable analyze plan --- ...plan-resource-bounded-resumable-analyze.md | 1698 +++++++++++++++++ 1 file changed, 1698 insertions(+) create mode 100644 docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md diff --git a/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md b/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md new file mode 100644 index 000000000..8cdd91d2a --- /dev/null +++ b/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md @@ -0,0 +1,1698 @@ +# GitNexus Engineering Plan — resource-bounded, resumable analyze + +> Task: implement [GitNexus #3598](https://github.com/abhigyanpatwari/GitNexus/issues/3598), without performance degradation. +> Depth: Deep; form: full; impact depth: 3. Planning only; implementation requires the LFG plan gate. +> Evidence verified at commit f641342d1fcbfdcb764c5e478db0f69e6cea899a on fix/ci-timing-download-retries. +> Index commit matches HEAD, but analyzer provenance is not established against current source; graph symbol identities also disagree with source. **Stale analyzer provenance — source-weighted limitation.** PDG unavailable. No refresh or build performed. +> Evidence provenance schema 2; global dirty digest 305f776a6b7dae0c761b813156861754467d5410fce87ece8bda66d0ea6d377c; 58 sorted cited paths; exact generated plan path excluded. + +## 1. Objective + +Deliver a verified graph and keyword index before optional embeddings finish. Bound retained embedding inputs and queued work by resources, resume paid work after interruption, and report graph and vector readiness independently. Preserve throughput, query latency, embedding text/coverage, and ordinary analysis performance. Memory savings obtained by silently doing less work do not satisfy the request. + +The acceptance scenarios are: bounded large-corpus input retention; the new graph queryable during and after embedding failure; unchanged-generation retries processing only unfinished units; safe recovery at every publication/write boundary; selective invalidation with reasons; truthful status/query capabilities; independent phase progress; and a short synthetic regression suite plus measured before/after performance. + +## 2. Current Behaviour + +- [verified] `queryEmbeddableNodes` reads each eligible label into `allNodes`, including content; the File fallback also materializes its result. `runEmbeddingPipeline` retains nodes, an identity set, hashes, stale IDs, and per-batch expanded texts before sub-batch embedding (`gitnexus/src/core/embeddings/embedding-pipeline.ts:162–277, 606–615, 648–788, 844–865`). Small inference batches therefore do not bound corpus input retention. +- [verified] The pipeline announces an active window before mutation; the default window is 5,000 nodes rounded to batch boundaries. It replaces exact row IDs and removes all rows for nodes with failed chunks (`embedding-pipeline.ts:283–319, 595, 726–729, 775–785, 969–1044`). Preserve these correctness rules. +- [verified] `runFullAnalysisInner` completes graph/FTS work and graph integrity checks before embeddings, but closes and publishes the database and freshness metadata only afterward (`gitnexus/src/core/run-analyze.ts:4759–4782, 4977–5044, 5434–5536`). The shared-store publication wrapper also waits for the inner analysis to return (`run-analyze.ts:1528–1541`). +- [verified] The analyze checkpoint callback drains WAL before recording completed windows. Staged recovery validates a retained staging family and filters unsafe node groups; it is useful existing behavior, not a missing feature (`run-analyze.ts:4995–5041`; `gitnexus/src/storage/embedding-recovery.ts:50–112`; `gitnexus/src/core/embeddings/staged-embedding-recovery.ts:117–185`). +- [verified] Checkpoints record model/dimensions/provider, pending IDs and optional staging/schema identity, but do not contain a complete persisted work manifest keyed to source coverage and analyzer identity (`gitnexus/src/storage/repo-meta.ts:47–53, 502–549`). +- [verified] `getIndexIncompleteReasons` includes embedding pending state, and status requires this entire list to be empty for current metadata. The MCP vector lane probes row counts and queries the graph database without using a completed-vector-generation receipt (`gitnexus/src/core/index-freshness.ts:134–175`; `gitnexus/src/cli/status.ts:321–330`; `gitnexus/src/mcp/local/local-backend.ts:4348–4478`). +- [verified] Analyze maps the embedding percentage to 90–98% overall. SSE correctly waits for the job's terminal status rather than trusting the pipeline's `ready` event (`run-analyze.ts:4980–4988`; `gitnexus/src/server/sse-progress.ts:60–75`). + +## 3. Relevant Architecture + +- [verified] Analyze owns an exclusive index lock; graph writes use the singleton adapter and its checkpoint driver. The read pool opens separate read-only native databases. Its injected in-process database mechanism does not establish cross-process read/write support (`run-analyze.ts:1451–1491`; `gitnexus/src/core/lbug/pool-adapter.ts:658–714, 993–1019`). +- [verified] `streamQuery` explicitly refuses to run with the singleton WAL driver active. Wrapping the current all-content query in this function while embedding writes continue would violate an existing invariant (`gitnexus/src/core/lbug/lbug-adapter.ts:1959–2004`). +- [verified] Vector results already return node IDs and are then hydrated through graph queries. A separate vector database can preserve this lookup shape (`local-backend.ts:4406–4426, 4485–4515`). Native vector-index creation still uses adapter-specific behavior, so changing only the injected query function is insufficient. +- [verified] Shared graphs are immutable, and pending embeddings currently prevent their publication. `resolveGraphPath` recognizes shared-store commit paths; local storage defaults to `lbug` (`gitnexus/src/core/shared-store-analyze.ts:377–407`; `gitnexus/src/storage/shared-store.ts:264–282`). +- [verified] `--memory-budget` is explicitly a main-thread V8 heap limit, with independent worker heaps. Native pools and embedding threads have separate sizing (`gitnexus/src/cli/index.ts:175–179`; `gitnexus/src/cli/analyze.ts:623–660`; `gitnexus/src/core/ingestion/workers/worker-pool.ts:691–702, 1035–1085`; `gitnexus/src/core/lbug/lbug-config.ts:479–513`; `gitnexus/src/core/embeddings/config.ts:74–109`). +- [verified] The three production embedding callers are analyze, `embeddingsSyncCommand`, and the `/api/embed` handler. Their checkpoint orchestration is duplicated (`run-analyze.ts:4977`; `gitnexus/src/cli/embeddings-sync.ts:197–245`; `gitnexus/src/server/api.ts:2278–2337`). + +## 4. GitNexus Findings + +All calls used `repo: "/workspace"`. The overview records HEAD f641342… and a 1.6.12 npm-distribution analyzer running Node 24.19.0. The local runner resolves to `npx gitnexus@latest`; there is no global executable. This session's Node is 22.23.3. Same package version is not proof of matching analyzer bytes. Refresh was skipped because provenance is unverified and the local graph is held by another process; no process was stopped. + +| Tool and target | Result | Consequence | +| --- | --- | --- | +| `query(search_query: "embedding pipeline checkpoint staged graph publication runAnalyze")` | Definitions led to embedding and freshness modules; no relevant process chains | Use targeted source reads for execution order. | +| `context(runEmbeddingPipeline, file_path: .../embedding-pipeline.ts)` | Found lines 581–1106; one incoming integration-test file | Production dynamic-import callers are missing from the graph result. | +| `impact(runEmbeddingPipeline, file_path: ..., upstream, maxDepth: 3)` | **UNKNOWN**, “No callers resolved” | Text references establish analyze, sync, and server callers; zero is not an all-clear. | +| `impact(runFullAnalysisInner, upstream, maxDepth: 3)` | **CRITICAL**, 8 impacted; d=1: `runFullAnalysis`, `runNative`, `runViaPool` | Returned UID names a benchmark `gc` function. Source confirms only `runFullAnalysis`; benchmark references are calls to `gc`, not this coordinator. Do not edit those benchmarks. | +| `impact(decideEmbeddingResume, file_path: .../embedding-checkpoint.ts, upstream, maxDepth: 3)` | **HIGH**, d=1: `runFullAnalysisInner`, `embeddingsSyncCommand`, `createServer` | All three production entrypoints need recovery parity tests. | +| `impact(semanticSearch, file_path: .../local-backend.ts, upstream, maxDepth: 3)` | **CRITICAL**, d=1: `_queryImpl`; 11 reported affected processes | Source confirms `_queryImpl` at line 3429. Several deeper process names are inconsistent with source; treat expansion as untrusted. | +| `impact(resolveBudgetHeap, file_path: .../cli/analyze.ts, upstream, maxDepth: 3)` | LOW; d=1: `ensureHeap` | Preserve its heap-only contract and test CLI respawn/watch propagation. | + +[graph] Deep-mode overview, clusters and processes resources were read; bounded summaries identify Storage, Lbug, Ingestion and Cli areas. They are not exhaustive dependency inventories. `context(runAnalyze, .../run-analyze.ts)` was empty because the real coordinator is `runFullAnalysisInner`; `context(publishSharedGraph, .../storage/shared-store.ts)` was empty because source locates it in `core/shared-store-analyze.ts`. + +## 5. Statement-Level PDG Findings + +`pdg_query(mode: "controls", target: "gitnexus/src/core/embeddings/embedding-pipeline.ts")` returned **“no PDG layer”**. No CDG/REACHING_DEF edges are claimed. A build-current, isolated index refresh with `--index-only --pdg` belongs to execution, before editing central symbols. + +These are **source-derived ordering constraints**, not reconstructed PDG edges: + +1. [verified] Validate identities before model requests or deleting rows (`embedding-pipeline.ts:606–615`). Streaming must retain whole-manifest identity validation before mutation, using disk-backed validation when necessary. +2. [verified] Persist the active-window intent before delete/insert; checkpoint durable rows before completed progress (`embedding-pipeline.ts:775–785`; `run-analyze.ts:4995–5041`). +3. [verified] A current content hash on one chunk does not prove the complete chunk set survived. A failed node is removed as a whole (`embedding-pipeline.ts:992–1014`). +4. [verified] The database must be durable and published before advancing metadata; a collapse verdict aborts staged publication (`run-analyze.ts:4763–4774, 5434–5521`). Moving publication earlier must carry these checks with it. +5. [verified] Expanding all chunks of a node batch before inference can exceed an input-byte budget even with a bounded node count (`embedding-pipeline.ts:844–865`). Both content loading and chunk expansion require backpressure. +6. [inferred] Rewriting megabyte-scale `RepoMeta` after every small unit, repeatedly scanning labels for pagination, or rebuilding HNSW per unit could erase the throughput benefit. `saveMeta` serializes the whole metadata document and its legacy mirror (`gitnexus/src/storage/repo-manager.ts:323–333`). Use a compact job journal for frequent updates. + +## 6. Proposed Changes + +The following are design decisions to implement and measure, not claims that the new behavior exists. + +### A. Separate graph publication from vector work + +In `runFullAnalysisInner` and `runFullAnalysis`, extract a graph-finalization boundary into a new `core/analysis-publication.ts` helper. It owns graph identity/integrity verification, FTS outcome, durable native close, graph/metadata publication, registry registration and parse-cache publication. Execute this boundary before loading the embedding model. Preserve legitimate FTS opt-outs/degraded states; never label unavailable keyword search as available. + +Use a versioned, compact publication receipt to reconcile a crash between graph-file publication and metadata publication. Record intended generation, source-coverage digest, analyzer identity and schema identity before switching the graph. Recovery either finishes publication of a verified generation or retains the prior one; it must not trust a newer timestamp alone. New generation filenames allow Windows readers to retain old handles without requiring overwrite of an open file. Extend `resolveGraphPath` and storage validation to accept only an owned, validated generation path; legacy local `lbug` and existing shared-store paths remain readable. No cross-slot paths or symlink escapes. + +Create a **separate vector database per generation**, using the existing LadybugDB vector schema and index format, behind a new `core/embeddings/vector-store.ts`. Its writer exclusively owns an unpublished file; graph readers open the already published graph. Once vector coverage is verified and the vector file is checkpointed/closed, atomically publish a small pointer that binds it to the graph generation. Readers never open the active vector writer's file. A graph-preserving embedding failure cannot roll back or mark that graph as an incomplete graph write. + +This avoids a second full-graph copy for every embedding job and avoids changing the native engine's locking model. It adds a second database handle during embeddings and vector queries; account for its pool in the resource budget and measure its cost. Keep graph-only runs on their current fast path. Do not build graph tables/FTS inside the vector-only database. Do not pass its handle through the graph singleton or inadvertently allocate two default-sized native pools. + +Update `publishSharedGraph` to publish verified graph facts independently of vector debt. Keep pending vector work checkout-owned; completed immutable vector artifacts may be reused only under a matching identity. Update store seeding, pointer validation and reclamation together; a pending job pins its graph and vector artifacts. A sibling checkout must never clean or mutate them. + +### B. Persist a deterministic, bounded work manifest + +Replace the all-content reader inside `runEmbeddingPipeline` with a new `core/embeddings/embedding-work-manifest.ts` source. For each graph generation, build one durable manifest in label/id order containing identifiers, source bounds, selection information and unit boundaries. Use a key-only ordered scan plus bounded primary-key fetches; evaluate its native query plan first. If ordering cannot stay within the native memory budget, use bounded sorted runs on disk. Never use growing OFFSET pagination or a full content query per page. + +Preflight duplicate/invalid IDs across the entire manifest before any delete or provider call. Keep the File fallback only when all eligible symbol labels are empty; a failed label read remains an error. Persist the selection receipt so resume does not re-enumerate completed units. Store identifiers/fingerprints rather than a second copy of raw source in the manifest. Read content from the pinned graph generation, respecting its full/symbol/none retention policy; never substitute changing working-tree text during a job. + +Bound units by input bytes, expected vector bytes and node count. Bound chunk text, update metadata, encoded HTTP bodies, returned vectors and prefetch together; prefetch at most one additional unit and retain one active durable write unit. Large nodes must use bounded content/chunk production preserving current text, overlap, Unicode and line semantics. A node too large for the supported AST working set must take an existing semantically valid bounded path, or fail explicitly before mutation; never silently truncate or claim completion. Inability to handle the synthetic oversized case remains a failed acceptance criterion, not a permitted corpus omission. + +Move corpus-size identity/hash/pending maps out of the live work queue: use manifest records and indexed vector-store lookups in bounded batches. A small-corpus in-memory optimization is permitted only under the same byte limit and restart semantics. Hash each changed node once. Reuse the existing prepared/batched writes; do not replace them with one new query preparation per row. + +### C. One durable job protocol for all embedding callers + +Add `core/embeddings/embedding-job.ts` and a filesystem-only `storage/embedding-job.ts` journal. Adapt `runEmbeddingPipeline`, `decideEmbeddingResume`, analyze, embeddings sync and `/api/embed` to that protocol. Preserve existing public function signatures with optional adapter/options additions until all callers migrate; the completed feature must leave no unbounded production entrypoint. + +Job identity includes graph generation and covered-content digest (dirty worktrees included), source revision, analyzer/parser identity, graph and vector schema fingerprints, provider/model/dimensions, text/chunking identity, and effective text-producing options. Do not include credentials or secret-derived identity data. Batch size, worker count and memory budget are scheduling choices: lowering them must not invalidate already completed vectors or alter persisted unit identity. + +For each unit: (1) fsync active intent; (2) replace all chunk rows for affected nodes; (3) validate complete per-node chunk counts/indices and hashes; (4) drain/checkpoint native writes; (5) atomically persist/fsync the completed cursor and counts; (6) advance. One unit may be active at a time. A crash after durable rows but before cursor publication may replay that one unit idempotently. Previously acknowledged units must not call the provider again. Partial failures remain durable pending records, with no successful coverage claim. + +Use a compact cursor plus append-only/segmented failed-unit records, not a growing pending-ID array rewritten with full repository metadata each window. Validate journal checksums/version and artifact ownership. On journal loss/corruption, preserve artifacts, derive only provable complete units, and log uncertainty; do not accept isolated matching hashes as completion. + +An unchanged compatible graph takes an embedding-only resume path before parse/resolve/graph-write work. For a source/analyzer/schema change, validate the graph anew but reuse vectors whose complete chunk sets and embedding-input fingerprints still match. Provider/model/dimension or text changes invalidate the affected vector space/input set, not an otherwise valid graph. Explain invalidation before requests. Preserve explicit `--drop-embeddings` and existing `--force` meaning; a forced rebuild may discard scheduling receipts while still preserving provably reusable embeddings as existing recovery tests require. + +Legacy monolithic indexes remain readable. Migrate their complete vector groups through the existing bounded spill path only when a vector write is needed; never re-embed merely to relocate storage. Keep the old readable artifact until publication succeeds. Legacy interrupted checkpoints retain their strict validation; do not infer a generation identity they never recorded. Avoid changing the graph schema fingerprint solely for vector storage: `EMBEDDING_SCHEMA` is already excluded (`gitnexus/src/core/lbug/schema.ts:719–729, 819–835`). Validate downgrade behavior explicitly: new metadata must never coexist with an obsolete canonical `lbug` that an older binary could mistake for the current graph. Retain rollback artifacts under generation-specific names and document the required rebuild for binaries that do not support the new format. + +### D. Budget the full resource envelope without changing existing flag semantics + +Add a pure `core/analysis-resource-budget.ts` resolver and a new `--resource-budget ` total-budget option. Preserve `--memory-budget` as the main-heap override. Resolve once at the entrypoint and pass explicit immutable budgets through analyze and embedding adapters; avoid global configuration that leaks between server jobs. The automatic total ceiling uses effective process/container memory with headroom; record the selected allocations. + +Account for main heap and native graph working set, worker V8/native allocations, graph and vector native pools, ONNX/sidecar runtime, byte queues, buffers and headroom. Count worker RSS correctly: worker threads share process RSS, while a model subprocess must be counted separately. Reuse `effectiveRamBytes`, CPU limits, existing pool safety floors and worker drain behavior. Release parser capacity before embedding; reuse the released budget instead of reserving all phases simultaneously. Reject impossible explicit combinations before destructive work; do not silently reinterpret an unbounded native-pool override as a compliant hard cap. + +Retain existing concurrency when it fits. Use bounded parallel input preparation/inference only when CPU, provider policy and memory allow; serialize database mutations and cursor advancement. Backpressure must pause admission rather than kill a worker inside native code. A configured application budget cannot promise an OS-enforced RSS ceiling for arbitrary native allocations: expose reservations and measured peak, include cgroup stress validation, and fail clearly when minimum viable native/model allocations do not fit. + +### E. Readiness and progress become separate, compatible fields + +Extend `RepoMeta` with versioned graph/vector generation receipts and a compact job reference. Keep the existing aggregate `incompleteReasons` for compatibility, adding graph-specific completeness/freshness and embedding progress fields. A pending embedding job must not force a graph rebuild or hide the newly current graph. Keep existing vector capability status strings: pending/failed/unverified vector coverage reports `unavailable` with a reason; only a complete artifact reports `vector-index`, or `exact-scan` when complete coverage exists but HNSW is unavailable. Retry exhaustion remains visible as failed/incomplete vector work. + +Gate the MCP vector lane **before** model initialization or vector-table queries. Resolve and pin a matching published vector generation; hydrate IDs from that same graph generation. Generation mismatch returns graph/keyword results with the reason. Preserve concurrent BM25/vector dispatch and exact-scan limits. Update the exported semantic-search path used outside MCP as well as `LocalBackend.semanticSearch`; do not leave a server-only route reading stale in-graph vectors. + +Extend `EmbeddingProgress` and server job progress with graph phase, embedding phase, unit totals/completed/reused/pending/failed, active unit and known node/chunk counts. Define categories without double counting: completed (new) + reused + pending + failed = total; active is a subset of pending. Completion advances only after durable acknowledgement. Retried failures move categories explicitly. Count unknown totals as unknown until the manifest is ready; model download is a separate indicator. CLI prints graph completion and an independent embedding fraction. SSE remains open until job terminality, and the public serializer must explicitly copy only safe numeric/enumerated progress fields (`gitnexus/src/server/ops-snapshot.ts:227–234`). + +## 7. Implementation Sequence + +Each step ends with focused checks and a coherent commit. Steps 2–5 provide inert helpers/adapters until the integration switch in step 6; do not expose half-migrated production behavior. + +1. **Re-anchor and establish the performance baseline.** Use an isolated checkout because this tree has unrelated edits. Build current source with `npm run build`; index that checkout with its explicit local `dist/cli/index.js analyze --index-only --pdg`. Re-run impacts and bounded PDG slices. Investigate graph UID mismatches if still present. Add a short corpus/measurement harness and real-native feasibility tests for separate graph readers, vector writer, generation publication, bounded manifest reads and Windows open handles. Compare the proposed reader/writer path to current behavior before committing to integration. Route back to this plan if the architecture cannot meet the performance requirement. +2. **Add resource contracts and manifest reader.** Tests cover compatibility of heap-only flags, total-budget accounting, stable units, failed reads, duplicate IDs, and maximum live bytes. Keep existing parse algorithms and language-provider hooks unchanged. +3. **Add vector generation storage and durable journal.** Implement exclusive vector ownership, compact job persistence, one-active-unit acknowledgement, complete chunk receipts and safe artifact cleanup. Test all interruption boundaries with real native storage and a deterministic fake provider. +4. **Adapt the embedding pipeline.** Stream content/chunks through bounded units, preserve rejection isolation and all-or-none node recovery, perform bounded cache lookups, and build HNSW once at completion. Verify embedding-input fingerprints and row parity against the baseline. +5. **Add graph/vector publication and reader compatibility.** Implement prepared/publication recovery, safe generation resolution, legacy lookup, query gating and per-artifact pools. Validate both warm and newly started readers while a vector writer is held open. +6. **Integrate all writers and surfaces.** Publish graph/metadata/registry/cache before embedding; add the embedding-only resume branch; use the same job runner for sync and server. Wire shared-store pinning/cleanup, status and phase progress. Enable the new path only when these consumers work together. Keep graph-only and unchanged no-op paths free of model/vector/spool initialization. +7. **Run the regression and performance matrix; document behavior.** Run focused and full checks below. Record raw paired samples and environment, compare total and phase times, peak memory, query latency and retry request counts. Update RUNBOOK and CLI/resource-budget documentation. Regenerate intentional goldens once at the end; do not replace baselines to conceal regressions. +8. **Graph change analysis and review.** Run `detect_changes(scope: "all")` before each commit; partial/truncated results are unresolved. After implementation refresh the graph and invoke `gitnexus-review` on the implementation branch, separately including local changes if any. Do not push or open a PR automatically. + +## 8. Test Strategy + +Existing tests to extend (located at the pinned source; representative pipeline, memory and native recovery cases source-read): + +| Existing file | Required regression coverage | +| --- | --- | +| `gitnexus/test/unit/embedding-pipeline.test.ts` | Current-hash interrupted windows, window-before-write order, exact-key replacement, input rejection isolation, File fallback; assert bounded content AND expanded text/vector bytes. | +| `gitnexus/test/unit/embedding-checkpoint.test.ts` | Identity changes, legacy kinds, force/drop compatibility, failed retry exhaustion remaining observable. | +| `gitnexus/test/unit/analyze-memory-budget.test.ts` | Existing heap-only CLI contract remains; add total-budget conflicts and watch propagation in a new resource-budget test. | +| `gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts` | Keep durable-group reuse, second-crash recovery, changed/deleted nodes, model mismatch, and explicit drop. Existing suite skips Windows; new publication tests must cover it. | +| `gitnexus/test/integration/embedding-identity-integrity.test.ts` | Identity validity, unique composite keys, complete chunk sets and cold reopen. | +| `gitnexus/test/unit/local-backend-lazy-vector.test.ts` | Pending/mismatched artifacts cause zero vector/model requests; complete artifact keeps lazy loading and ranking. | +| `gitnexus/test/unit/server-sse-payload.test.ts` | Safe additive progress fields; graph-ready is nonterminal; exactly one final event reflecting failures. | +| `gitnexus/test/integration/shared-store-embedding-reuse.test.ts` | A sibling's graph/vector publication and cleanup cannot lose durable work. | + +New tests, with concrete scenarios: + +- `test/unit/embedding-work-manifest.test.ts`: many small nodes, few huge nodes, Unicode, duplicate IDs on different pages, label-read failure, deleted nodes, low budget, budget change on retry → stable manifest identity, complete text coverage, bounded live bytes, no provider calls before identity preflight. +- `test/unit/analysis-resource-budget.test.ts`: cgroup smaller than host, explicit heap/worker/native overrides, graph/vector phases, tiny budgets → allocations fit or fail before mutation; worker concurrency is preserved when feasible. +- `test/integration/analyze-phase-publication.test.ts`: stop before graph publish → old graph; stop immediately after it → new graph/FTS available; stop between file and metadata publication → recovery reconciles the intended generation without exposing a half graph. Test cold process readers and continuously querying warm readers. +- `test/integration/embedding-job-restart.test.ts`: kill a child before active intent, after intent, during a multi-chunk write, after WAL durability but before completed cursor, after cursor, during HNSW build, and after vector-file publication before pointer publication → at most the active unit replayed, acknowledged units submit zero requests, complete final chunk sets with no duplicates. Repeated crashes, ENOSPC, checkpoint failure, journal corruption and cancellation preserve the published graph. +- `test/integration/embedding-resource-bounds.test.ts`: lazy synthetic 10k/100k/1m-node input descriptors and controlled content payloads → the queue high-water mark is capped independently of corpus size. Do not allocate the complete fixture in the test process. Use a smaller real-native case to verify the reader's allocations and a large-node case to expose expansion, not just node-count bounds. +- `test/integration/vector-generation-compatibility.test.ts`: legacy monolithic index, interrupted legacy checkpoint, local/shared/external/branch storage, Windows open reader, simultaneous sync/analyze, model/dimension change, no FTS/VECTOR extension → correct ownership, graph availability, honest vector mode and stable existing result shapes. + +Execution commands, run under `gitnexus/` (scripts verified in package.json): + +```sh +npm run build +npx tsc --noEmit +npm test -- test/unit/embedding-pipeline.test.ts test/unit/embedding-checkpoint.test.ts test/unit/analyze-memory-budget.test.ts test/unit/local-backend-lazy-vector.test.ts test/unit/server-sse-payload.test.ts +npm run test:integration -- test/integration/analyze-staged-embedding-recovery.test.ts test/integration/embedding-identity-integrity.test.ts test/integration/shared-store-embedding-reuse.test.ts +npm test +node --import tsx bench/embedding-inputs/measure.cjs --check +node --import tsx bench/embedding-identities/measure.cjs --check +``` + +The build is necessary for shared-package output and real CLI integration tests. Add the newly created tests to the focused commands. Introduce `bench/analyze-resume/measure.mjs` in step 1 with documented arguments; that command is **proposed**, not an existing runnable tool. Do not run it before creation. Planning ran no tests or benchmarks. + +Performance protocol: compare the same corpus, source features, Node/native versions, hardware, model/provider, request limits and resource budget, using separate isolated stores. Alternate baseline/candidate order with warmup and at least seven measured pairs; increase samples if noisy. Include cold graph-only analyze, warm no-op, tiny incremental edit, all-cached vectors, fresh embeddings, local inference, a deterministic fast HTTP provider, interrupted resume, and warm/cold graph+keyword+vector queries during embedding. Measure total elapsed time, phase durations, nodes/chunks per second, p50/p95 query latency, CPU, process plus sidecar peak RSS, live queue bytes, native pool reservations, disk bytes and provider requests. Include index/manifest creation, migration and final HNSW time; do not hide costs outside the timer. Preserve ranking and input/output fingerprints. + +**Performance acceptance:** no repeatable throughput loss or query-latency regression against the pinned baseline at comparable resources. Do not translate the user's requirement into an arbitrary allowed slowdown. Publish raw paired samples and uncertainty; inconclusive measurements are unresolved, not a pass. A stricter optional memory budget naturally changes available resources: compare both implementations under that same budget and separately check default behavior. Any repeatable degradation routes back to tuning/design before shipment. + +## 9. Risk and Impact Analysis + +- **CRITICAL coordinator / query path, HIGH resume policy:** no code edit is authorized by this planning artifact alone. The LFG proceed decision should explicitly acknowledge these disclosed changes, satisfying GUARDRAILS maintainer sign-off. Recheck impact on the actual implementation checkout. +- **Every reported d=1 dependent is accounted for:** `runFullAnalysis` gets graph-boundary integration; `runNative`/`runViaPool` are rejected graph false positives after source search; analyze/sync/server resume callers get parity tests; `_queryImpl` gets vector gating/ranking tests; `ensureHeap` gets old-flag/new-budget compatibility tests. Missing dynamic-import callers were added through source verification. +- **Publication and native safety:** data-before-metadata ordering, generation pinning, WAL driver shutdown, no native-thread termination, and Windows handles are mandatory. Keep current collapse and scope-incompleteness diagnostics. A generation identifier alone does not certify contents. +- **Storage/migration:** graph schema and vector schema have separate validity; don't force graph rebuilds for vector-only changes. No destructive legacy migration. Keep source artifacts until verified publication. Two native handles and on-disk manifests cost resources; measurements decide acceptability. +- **Partial vectors:** matching row hashes, exhausted retries, `ready`, or successful HNSW creation are not proof of full coverage. New pending/failed jobs cannot appear as complete vector capability. +- **Graph availability:** validate graph queries through both MCP and HTTP, including fresh processes, while the vector write handle remains open. Existing readers continuing to see only the old graph does not satisfy #3598. +- **Scope:** CLI/core/storage/server/tests plus necessary benchmark/documentation files only. No language-resolution changes, dependency upgrades, CI/release/security-configuration changes, or edits to existing dirty files. External storage ownership and public-progress redaction remain intact. + +## 10. Files Expected to Change + +Existing symbol names below were source-verified; all proposed new files/symbols are explicitly new. + +| File(s) | Responsibility | +| --- | --- | +| `gitnexus/src/core/run-analyze.ts` | `runFullAnalysis`, `runFullAnalysisInner`: graph publication and embedding-only resume. | +| `gitnexus/src/core/embeddings/embedding-pipeline.ts`, `types.ts`, `config.ts` | `runEmbeddingPipeline`, internal readers, `EmbeddingProgress`, `resolveEmbeddingConfig`: bounded units, adapter dependencies, safe progress. | +| `gitnexus/src/core/embedding-checkpoint.ts` | `decideEmbeddingResume` and checkpoint factories: shared protocol and legacy compatibility. | +| `gitnexus/src/storage/repo-meta.ts`, `embedding-recovery.ts` | `RepoMeta`, recovery reference validation: versioned generation/job receipts. | +| `gitnexus/src/storage/shared-store.ts`, `storage-resolver.ts`; `gitnexus/src/core/shared-store-analyze.ts` | Generation resolution, owned-path validation, `publishSharedGraph`/seeding and pinned-artifact cleanup integration. | +| `gitnexus/src/core/lbug/pool-adapter.ts`, `lbug-adapter.ts`, `lbug-config.ts` | Distinct read/write adapter lifetimes, bounded native reads, explicit per-handle pool reservations; retain legacy callers. | +| `gitnexus/src/core/index-freshness.ts`; `gitnexus/src/cli/status.ts` | Graph freshness independent from embedding debt. | +| `gitnexus/src/cli/index.ts`, `analyze.ts`, `embeddings-sync.ts` | New total-resource flag, existing heap contract, common embedding runner and phase output. | +| `gitnexus/src/core/ingestion/workers/worker-pool.ts` | Pass budget allocation to pool sizing/drain; preserve native-safe lifecycle. | +| `gitnexus/src/mcp/local/local-backend.ts`; `gitnexus/src/server/api.ts` | Matching vector-generation reads, all writer entrypoints and HTTP progress. | +| `gitnexus/src/server/analyze-job.ts`, `ops-snapshot.ts` | Optional typed phase counters and explicit safe serialization. | +| **New:** `gitnexus/src/core/analysis-publication.ts`, `analysis-resource-budget.ts` | Publication protocol and pure budget resolution. | +| **New:** `gitnexus/src/core/embeddings/embedding-work-manifest.ts`, `embedding-job.ts`, `vector-store.ts`; `gitnexus/src/storage/embedding-job.ts` | Bounded work source, execution protocol, vector-only adapter, filesystem-only journal. | +| Existing/new tests in §8; **new** `gitnexus/bench/analyze-resume/measure.mjs` | Crash matrix, performance and bounded-retention evidence. | +| `RUNBOOK.md` and relevant CLI documentation | Resource option, graph/vector status, recovery and measured limits. | + +This is a cross-module change, not a two-file fix. Additional caller wiring discovered by the fresh graph belongs in the execution drift report; architectural expansion returns to the plan gate. + +## 11. Reusable Implementation Context + +```json +{ + "implementation_context": { + "task_summary": "Implement #3598: resource-bounded analyze, early graph publication, durable embedding-only resume, truthful readiness/progress, with no performance degradation.", + "acceptance_criteria": [ + "New verified graph and keyword queries work during/after embedding failure.", + "Retained content, expanded chunks, vectors and prefetch fit an explicit resource envelope independently of corpus size.", + "On compatible retry, acknowledged units cause no provider requests; at most the active unit replays.", + "All crash boundaries converge to complete unique per-node chunk sets.", + "Invalidation is selective and explained; graph and vector identities are separate.", + "CLI/MCP/HTTP report graph freshness, embedding progress and actual vector readiness independently.", + "Synthetic scaling tests and paired default/constrained performance measurements show no repeatable regression." + ], + "evidence_provenance": { + "schema_version": 2, + "head_commit": "f641342d1fcbfdcb764c5e478db0f69e6cea899a", + "generated_plan_path": "docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md", + "global_dirty_digest": { + "algorithm": "sha256", + "canonicalization": "gitnexus-evidence-provenance-v2 NUL-framed UTF-8 records", + "value": "305f776a6b7dae0c761b813156861754467d5410fce87ece8bda66d0ea6d377c" + }, + "cited_path_manifest": [ + { + "path": "AGENTS.md", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", + "index_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", + "worktree_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", + "untracked_digest": "absent" + }, + { + "path": "CONTRIBUTING.md", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", + "index_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", + "worktree_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", + "untracked_digest": "absent" + }, + { + "path": "GUARDRAILS.md", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", + "index_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", + "worktree_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", + "untracked_digest": "absent" + }, + { + "path": "RUNBOOK.md", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", + "index_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", + "worktree_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/measure.mjs", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/embedding-identities/measure.cjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", + "index_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", + "worktree_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/embedding-inputs/measure.cjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", + "index_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", + "worktree_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/package.json", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", + "index_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", + "worktree_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/scripts/bench/fts-evict-reload-rss.mjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", + "index_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", + "worktree_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/cli/analyze.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", + "index_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", + "worktree_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/cli/embeddings-sync.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", + "index_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", + "worktree_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/cli/index.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", + "index_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", + "worktree_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/cli/status.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", + "index_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", + "worktree_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/analysis-publication.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/analysis-resource-budget.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embedding-checkpoint.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", + "index_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", + "worktree_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/config.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", + "index_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", + "worktree_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/embedding-identity.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", + "index_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", + "worktree_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/embedding-job.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/embedding-pipeline.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", + "index_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", + "worktree_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/staged-embedding-recovery.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", + "index_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", + "worktree_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/types.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", + "index_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", + "worktree_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/vector-store.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/index-freshness.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", + "index_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", + "worktree_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/ingestion/utils/effective-ram.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", + "index_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", + "worktree_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/ingestion/workers/worker-pool.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", + "index_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", + "worktree_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/lbug/lbug-adapter.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", + "index_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", + "worktree_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/lbug/lbug-config.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", + "index_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", + "worktree_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/lbug/pool-adapter.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", + "index_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", + "worktree_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/lbug/schema.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", + "index_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", + "worktree_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/run-analyze.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", + "index_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", + "worktree_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/shared-store-analyze.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", + "index_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", + "worktree_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/mcp/local/local-backend.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", + "index_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", + "worktree_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/analyze-job.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", + "index_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", + "worktree_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/api.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", + "index_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", + "worktree_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/ops-snapshot.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", + "index_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", + "worktree_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/sse-progress.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", + "index_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", + "worktree_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/embedding-job.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/embedding-recovery.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", + "index_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", + "worktree_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/repo-manager.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", + "index_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", + "worktree_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/repo-meta.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", + "index_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", + "worktree_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/shared-store.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", + "index_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", + "worktree_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/storage-resolver.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", + "index_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", + "worktree_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/analyze-phase-publication.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", + "index_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", + "worktree_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/embedding-identity-integrity.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", + "index_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", + "worktree_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/embedding-job-restart.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/embedding-resource-bounds.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", + "index_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", + "worktree_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/integration/vector-generation-compatibility.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/analysis-resource-budget.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/analyze-memory-budget.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", + "index_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", + "worktree_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-checkpoint.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", + "index_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", + "worktree_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-pipeline.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", + "index_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", + "worktree_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-work-manifest.test.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/local-backend-lazy-vector.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", + "index_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", + "worktree_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/server-sse-payload.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", + "index_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", + "worktree_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", + "untracked_digest": "absent" + } + ] + }, + "primary_symbols": [ + { + "symbol": "runFullAnalysisInner", + "file": "gitnexus/src/core/run-analyze.ts", + "lines": "1566–5794", + "role": "Graph and embedding phase orchestration; CRITICAL graph impact, IDs unreliable", + "source_verified": true + }, + { + "symbol": "runEmbeddingPipeline", + "file": "gitnexus/src/core/embeddings/embedding-pipeline.ts", + "lines": "581–1106", + "role": "Bounded source, checkpoint windows, complete chunk persistence; UNKNOWN upstream", + "source_verified": true + }, + { + "symbol": "decideEmbeddingResume", + "file": "gitnexus/src/core/embedding-checkpoint.ts", + "lines": "190–290", + "role": "Resume identity/flags and common caller policy; HIGH impact", + "source_verified": true + }, + { + "symbol": "LocalBackend.semanticSearch", + "file": "gitnexus/src/mcp/local/local-backend.ts", + "lines": "4348–4528", + "role": "Generation-ready vector lane and graph hydration; CRITICAL impact", + "source_verified": true + }, + { + "symbol": "resolveBudgetHeap", + "file": "gitnexus/src/cli/analyze.ts", + "lines": "623–660", + "role": "Preserve heap-only flag contract; LOW impact", + "source_verified": true + } + ], + "related_symbols": [ + { + "symbol": "runFullAnalysis", + "relationship": "CALLS runFullAnalysisInner", + "relevance": "exclusive index lock and shared-store publication" + }, + { + "symbol": "queryEmbeddableNodes", + "relationship": "called by runEmbeddingPipeline", + "relevance": "all-content materialization" + }, + { + "symbol": "contentHashForNode", + "relationship": "called by pipeline", + "relevance": "stable text identity; no semantic shortcuts" + }, + { + "symbol": "batchInsertEmbeddings", + "relationship": "called by pipeline", + "relevance": "validated exact-key replacement" + }, + { + "symbol": "buildVectorIndex", + "relationship": "called by pipeline", + "relevance": "adapter-specific HNSW operation; publish once complete" + }, + { + "symbol": "embeddingsSyncCommand", + "relationship": "calls pipeline and resume policy", + "relevance": "production parity" + }, + { + "symbol": "createServer", + "relationship": "contains /api/embed caller", + "relevance": "production parity and safe API progress" + }, + { + "symbol": "_queryImpl", + "relationship": "calls LocalBackend.semanticSearch", + "relevance": "concurrent keyword/vector lanes" + }, + { + "symbol": "getIndexIncompleteReasons", + "relationship": "status consumer", + "relevance": "aggregate legacy incomplete reasons" + }, + { + "symbol": "publishSharedGraph", + "relationship": "called by runFullAnalysis", + "relevance": "immutable store publication and pending vector ownership" + }, + { + "symbol": "resolveGraphPath", + "relationship": "storage path resolution", + "relevance": "safe generation/legacy compatibility" + }, + { + "symbol": "resolveEmbeddingRecovery", + "relationship": "legacy checkpoint validation", + "relevance": "owned staging file families" + }, + { + "symbol": "recoverStagedEmbeddings", + "relationship": "legacy recovery", + "relevance": "reuse complete groups with bounded vector spill" + }, + { + "symbol": "saveMeta", + "relationship": "metadata publication", + "relevance": "full metadata serialized; avoid per-unit rewrites" + }, + { + "symbol": "streamQuery", + "relationship": "native read adapter", + "relevance": "must not run with singleton WAL driver" + }, + { + "symbol": "ensureHeap", + "relationship": "calls resolveBudgetHeap", + "relevance": "CLI respawn compatibility" + }, + { + "symbol": "createWorkerPool", + "relationship": "resource consumer", + "relevance": "worker caps and native-safe lifecycle" + }, + { + "symbol": "estimateBufferPool", + "relationship": "native resource sizing", + "relevance": "preserve page-size/COPY floors" + }, + { + "symbol": "publicOpsProgress", + "relationship": "SSE serializer", + "relevance": "explicit safe optional fields" + }, + { + "symbol": "resolveEmbeddingIdentity", + "relationship": "resume identity", + "relevance": "secret-free provider/model/dimensions" + } + ], + "execution_path": [ + "Current: lock -> graph parse/resolve/write/FTS/verify -> embedding windows -> final close/swap/meta/registry/cache -> shared publication.", + "Proposed: lock -> verify compatible published graph/job for fast resume; otherwise build/verify/publish graph and metadata -> immutable graph reader + exclusive unpublished vector writer -> intent/write/checkpoint/ack units -> verify complete coverage/HNSW -> publish matching vector pointer." + ], + "pdg_constraints": [], + "pdg_availability": "No PDG layer. Source-derived constraints in section 5; no reconstructed PDG edges.", + "architectural_patterns": [ + { + "pattern": "One canonical writer lock", + "example_location": "gitnexus/src/core/run-analyze.ts:1451", + "usage_guidance": "Recheck storage/source identity under lock; never let sync/analyze/server race publication." + }, + { + "pattern": "Immutable published graph, separately owned unpublished vectors", + "example_location": "gitnexus/src/core/shared-store-analyze.ts:377", + "usage_guidance": "Extend existing immutable shared-store behavior to phase publication; budget separate native handles." + }, + { + "pattern": "Exact-key replacement and complete node groups", + "example_location": "gitnexus/src/core/embeddings/embedding-pipeline.ts:283", + "usage_guidance": "Retain idempotent replacement and chunk completeness checks." + }, + { + "pattern": "Lazy vector lane", + "example_location": "gitnexus/src/mcp/local/local-backend.ts:4359", + "usage_guidance": "Gate completeness before model import; keep keyword/graph lane available." + } + ], + "files_to_modify": [ + { + "file": "gitnexus/src/core/run-analyze.ts", + "symbols": [ + "runFullAnalysis", + "runFullAnalysisInner" + ], + "intended_change": "Publish verified graph before embedding; compatible embedding-only resume" + }, + { + "file": "gitnexus/src/core/embeddings/embedding-pipeline.ts", + "symbols": [ + "runEmbeddingPipeline", + "queryEmbeddableNodes", + "semanticSearch" + ], + "intended_change": "Bound source/chunk/vector work; adapter-backed vector reads and writes" + }, + { + "file": "gitnexus/src/core/embeddings/types.ts", + "symbols": [ + "EmbeddingProgress" + ], + "intended_change": "Add optional durable phase counters" + }, + { + "file": "gitnexus/src/core/embeddings/config.ts", + "symbols": [ + "resolveEmbeddingConfig" + ], + "intended_change": "Pass bounded scheduling options without changing text identity" + }, + { + "file": "gitnexus/src/core/embedding-checkpoint.ts", + "symbols": [ + "decideEmbeddingResume", + "mintInterruptedCheckpoint", + "mintPartialCheckpoint" + ], + "intended_change": "Shared recovery and legacy migration" + }, + { + "file": "gitnexus/src/storage/repo-meta.ts", + "symbols": [ + "RepoMeta", + "EmbeddingRecoveryReference" + ], + "intended_change": "Versioned generation receipts and job reference" + }, + { + "file": "gitnexus/src/storage/embedding-recovery.ts", + "symbols": [ + "resolveEmbeddingRecovery" + ], + "intended_change": "Validate legacy/new artifact ownership" + }, + { + "file": "gitnexus/src/storage/shared-store.ts", + "symbols": [ + "resolveGraphPath" + ], + "intended_change": "Resolve owned immutable generation or legacy path" + }, + { + "file": "gitnexus/src/storage/storage-resolver.ts", + "symbols": [ + "inspectCodeIndexDB" + ], + "intended_change": "Validate new generation resolution" + }, + { + "file": "gitnexus/src/core/shared-store-analyze.ts", + "symbols": [ + "publishSharedGraph" + ], + "intended_change": "Separate shared graph validity from checkout vector debt; seed/pin/reclaim consistently" + }, + { + "file": "gitnexus/src/core/lbug/pool-adapter.ts", + "symbols": [ + "initLbug" + ], + "intended_change": "Generation-pinned graph/vector readers with explicit native reservations" + }, + { + "file": "gitnexus/src/core/lbug/lbug-adapter.ts", + "symbols": [ + "streamQuery" + ], + "intended_change": "Bounded read adapter; preserve checkpoint-driver exclusion" + }, + { + "file": "gitnexus/src/core/lbug/lbug-config.ts", + "symbols": [ + "estimateBufferPool", + "setBufferPoolSizeHint" + ], + "intended_change": "Budget all concurrent native pools" + }, + { + "file": "gitnexus/src/core/index-freshness.ts", + "symbols": [ + "getIndexIncompleteReasons" + ], + "intended_change": "Separate graph completeness while preserving aggregate reasons" + }, + { + "file": "gitnexus/src/cli/status.ts", + "symbols": [], + "intended_change": "Expose graph freshness and vector/job readiness separately" + }, + { + "file": "gitnexus/src/cli/index.ts", + "symbols": [], + "intended_change": "Add total resource-budget flag; retain heap-only flag contract" + }, + { + "file": "gitnexus/src/cli/analyze.ts", + "symbols": [ + "resolveBudgetHeap", + "ensureHeap" + ], + "intended_change": "Budget propagation and phase-specific progress" + }, + { + "file": "gitnexus/src/cli/embeddings-sync.ts", + "symbols": [ + "embeddingsSyncCommand" + ], + "intended_change": "Use shared durable embedding job" + }, + { + "file": "gitnexus/src/core/ingestion/workers/worker-pool.ts", + "symbols": [ + "resolveWorkerHeapCapMb", + "createWorkerPool" + ], + "intended_change": "Respect explicit phase allocation; preserve safe drain" + }, + { + "file": "gitnexus/src/mcp/local/local-backend.ts", + "symbols": [ + "LocalBackend.semanticSearch" + ], + "intended_change": "Gate on complete matching vectors before model work; hydrate same-generation graph" + }, + { + "file": "gitnexus/src/server/api.ts", + "symbols": [], + "intended_change": "Use same embedding job and generation-aware vector search" + }, + { + "file": "gitnexus/src/server/analyze-job.ts", + "symbols": [ + "AnalyzeJobProgress" + ], + "intended_change": "Optional counters without new terminal status values" + }, + { + "file": "gitnexus/src/server/ops-snapshot.ts", + "symbols": [ + "publicOpsProgress" + ], + "intended_change": "Explicit safe numeric/enumerated progress serialization" + }, + { + "file": "gitnexus/src/core/analysis-publication.ts", + "symbols": [], + "intended_change": "NEW: graph publication and crash reconciliation" + }, + { + "file": "gitnexus/src/core/analysis-resource-budget.ts", + "symbols": [], + "intended_change": "NEW: pure resource allocation and admission policy" + }, + { + "file": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", + "symbols": [], + "intended_change": "NEW: durable deterministic work manifest and bounded source" + }, + { + "file": "gitnexus/src/core/embeddings/embedding-job.ts", + "symbols": [], + "intended_change": "NEW: shared resumable execution protocol" + }, + { + "file": "gitnexus/src/core/embeddings/vector-store.ts", + "symbols": [], + "intended_change": "NEW: isolated vector-only native adapter" + }, + { + "file": "gitnexus/src/storage/embedding-job.ts", + "symbols": [], + "intended_change": "NEW: filesystem-only compact job journal" + }, + { + "file": "gitnexus/bench/analyze-resume/measure.mjs", + "symbols": [], + "intended_change": "NEW: paired throughput/memory/restart/query harness" + }, + { + "file": "RUNBOOK.md", + "symbols": [], + "intended_change": "Document phase recovery, new resource option and limits" + } + ], + "tests": [ + { + "file": "gitnexus/test/unit/embedding-pipeline.test.ts", + "scenarios": [ + "Existing: interrupted-window hashes, before-write intent, fallback and rejection isolation, bounded expansion" + ] + }, + { + "file": "gitnexus/test/unit/embedding-checkpoint.test.ts", + "scenarios": [ + "Existing: identity/force/drop/legacy cases; visible failed retry exhaustion" + ] + }, + { + "file": "gitnexus/test/unit/analyze-memory-budget.test.ts", + "scenarios": [ + "Existing: heap-only CLI flag compatibility" + ] + }, + { + "file": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", + "scenarios": [ + "Existing: completed vector reuse, second crash, changed/deleted nodes, model mismatch and drop" + ] + }, + { + "file": "gitnexus/test/integration/embedding-identity-integrity.test.ts", + "scenarios": [ + "Existing: exact identities and complete unique chunk sets" + ] + }, + { + "file": "gitnexus/test/unit/local-backend-lazy-vector.test.ts", + "scenarios": [ + "Existing: pending/mismatch causes no model or vector requests" + ] + }, + { + "file": "gitnexus/test/unit/server-sse-payload.test.ts", + "scenarios": [ + "Existing: additive safe counters and exactly one honest terminal event" + ] + }, + { + "file": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", + "scenarios": [ + "Existing: safe sibling reuse/publication and pending-job pinning" + ] + }, + { + "file": "gitnexus/test/unit/embedding-work-manifest.test.ts", + "scenarios": [ + "NEW: lazy large corpus, duplicate IDs, failed labels, stable byte-bounded units" + ] + }, + { + "file": "gitnexus/test/unit/analysis-resource-budget.test.ts", + "scenarios": [ + "NEW: cgroups, overrides, native floors, phase allocations and watch propagation" + ] + }, + { + "file": "gitnexus/test/integration/analyze-phase-publication.test.ts", + "scenarios": [ + "NEW: before/after graph publication crashes and cold/warm reader availability" + ] + }, + { + "file": "gitnexus/test/integration/embedding-job-restart.test.ts", + "scenarios": [ + "NEW: interruption at intent/write/durable/cursor/HNSW/pointer boundaries; at most active unit replay" + ] + }, + { + "file": "gitnexus/test/integration/embedding-resource-bounds.test.ts", + "scenarios": [ + "NEW: lazy 10k/100k/1m descriptors and real-native bounded-retention check" + ] + }, + { + "file": "gitnexus/test/integration/vector-generation-compatibility.test.ts", + "scenarios": [ + "NEW: legacy/shared/external/branch storage; Windows; concurrent writers; missing extensions" + ] + } + ], + "verification_commands": [ + "cd gitnexus && npm run build", + "cd gitnexus && npx tsc --noEmit", + "cd gitnexus && npm test -- test/unit/embedding-pipeline.test.ts test/unit/embedding-checkpoint.test.ts test/unit/analyze-memory-budget.test.ts test/unit/local-backend-lazy-vector.test.ts test/unit/server-sse-payload.test.ts", + "cd gitnexus && npm run test:integration -- test/integration/analyze-staged-embedding-recovery.test.ts test/integration/embedding-identity-integrity.test.ts test/integration/shared-store-embedding-reuse.test.ts", + "cd gitnexus && npm test", + "cd gitnexus && node --import tsx bench/embedding-inputs/measure.cjs --check", + "cd gitnexus && node --import tsx bench/embedding-identities/measure.cjs --check" + ], + "risks": [ + "CRITICAL coordinator/query and HIGH resume impact require gate acknowledgement; graph expansion has UID inconsistencies.", + "Native reader/writer lifetime and publication crash windows; Windows handles.", + "Additional vector pool and manifest I/O may regress performance; measure before integration.", + "Large-node chunking and native query materialization may exceed budgets.", + "Legacy/shared-store compatibility; incomplete vectors must never appear complete." + ], + "assumptions": [ + "Prove separate vector database feasibility and throughput with real-native concurrent-reader tests at the same budget before integration.", + "Inspect native query plan and scaling to prove bounded manifest reads avoid repeated label scans and unbounded native content retention.", + "Verify large-node input equivalence and peak memory; route back if chunker redesign is required.", + "Verify Windows and local-model execution environments; disclose unavailable validation." + ], + "open_questions": [ + "Determine budget/queue/concurrency defaults from measurements; no performance claim is established.", + "Refresh graph with source-current analyzer in execution and confirm/fallback on symbol UID mismatches.", + "Validate downgrade behavior for new generation pointers before enabling the format." + ], + "avoid": [ + "Do not repeat full repository discovery; use this context and targeted drift checks.", + "Do not modify pre-existing CI/eval/benchmark changes or untracked tooling files.", + "Do not edit code before the LFG gate and fresh impact analysis; do not commit before complete detect_changes and required tests.", + "Do not interpret UNKNOWN or the inconsistent current graph as proof of low impact.", + "Do not replace memory-budget heap semantics or bypass native COPY safety floors.", + "Do not materialize corpus content/hash maps; do not use repeated OFFSET pagination.", + "Do not claim streaming JavaScript alone bounds native query allocations.", + "Do not terminate native-active workers or use the singleton streamQuery with its WAL driver active.", + "Do not publish partial vectors as complete, silently truncate content, or omit failed work.", + "Do not re-embed completed compatible units, copy the full graph for each embedding job, or rebuild HNSW per unit.", + "Do not discard legacy vectors before successful migration/publication.", + "Do not weaken performance acceptance, remove features, change baselines to mask regressions, or claim unmeasured results.", + "Do not alter CI/release/security config, push, or publish a PR as part of this plan." + ], + "index_refresh": { + "status": "skipped", + "reason": "Current index HEAD but analyzer-source equivalence unverified; graph UID mismatch; no PDG; another process owns local graph. Planning must not rebuild dist.", + "runner": "node .gitnexus/run.cjs -> npx gitnexus@latest", + "recorded_cli_version": "1.6.12", + "recorded_runtime": "v24.19.0", + "planning_runtime": "v22.23.3" + }, + "validation_status": "Planning source review only; no production changes, tests, benchmarks or commits performed." + } +} +``` + +## 12. Assumptions and Open Questions + +- [assumed] A vector-only LadybugDB plus a published read-only graph can meet current latency and throughput. Prove with real-native cold/warm concurrent-reader benchmarks in step 1; the additional pool must fit the same resource envelope. +- [assumed] A single key-only scan plus bounded content fetches can avoid repeated full scans and unbounded native materialization. Inspect the installed engine's query behavior and measure scaling. Do not assume `ORDER BY id LIMIT` uses an index or that JavaScript streaming bounds native allocations. +- [assumed] Very large single-node chunking can preserve embedding input semantics within the configured envelope. Test actual chunker behavior before claiming compliance; if it requires a broader chunker redesign, deepen this plan rather than silently truncating or widening the budget. +- [assumed] Linux and Windows runners, plus a representative local embedding runtime, are available during execution. If unavailable, record the platform/runtime gap and do not declare its acceptance tests passed. +- **Open implementation gate:** exact budget defaults, queue byte caps and overlap/concurrency settings must come from measurements; preserve existing useful concurrency whenever memory permits. No performance improvement or non-regression has been measured during planning. +- **Evidence limitation:** matching graph HEAD does not repair the npm analyzer identity mismatch or inconsistent graph UIDs. Source is authoritative. Build-current graph/PDG verification must precede implementation; if the mismatch persists, record source-derived impact plus unresolved graph coverage. +- **Compatibility decision:** introduce `--resource-budget` for the whole analysis process family while preserving `--memory-budget`. Public metadata additions remain optional; legacy status aggregation remains available. Downgrade binaries that do not understand new generation pointers must fail clearly or use the retained legacy graph, never manufacture current freshness. +- **Working tree:** `.github/scripts/evolution-runner.py`, `eval/tests/test_evolution_runner.py`, and `gitnexus/bench/analyze-phase-breakdown.md` were already dirty; untracked tooling files also existed. They are outside this task. The evidence snapshot records them without including their contents in this plan. +- **Deferred:** distributed embedding workers, new database engines, provider-specific acceleration, full graph-ingestion out-of-core redesign, partial-vector serving, automatic background scheduling, and unrelated graph-identity repairs. They are not required to implement this phase boundary; if resource validation proves one necessary, re-plan explicitly. + +## 13. Definition of Done + +- [ ] Graph/keyword publication is independently durable and visible to new readers before embedding completes; failures preserve that new graph. +- [ ] At fixed configuration, retained input/chunk/vector queues remain bounded as corpus size grows, including large nodes and failed work; total-resource reservations and observed memory are reported honestly. +- [ ] Compatible retries skip parse/graph rebuilding and completed embedding requests; only unfinished work plus at most the active interrupted unit is replayed. +- [ ] Every crash boundary passes with exact complete chunk sets, unique keys and no lost acknowledged work; invalidation reasons explain retained versus recomputed work. +- [ ] Graph freshness, embedding coverage and vector readiness are distinct across CLI, MCP, HTTP and SSE; pending/failed vectors cannot masquerade as complete. +- [ ] Legacy indexes/checkpoints, explicit force/drop, shared stores, external/branch storage and Windows readers retain safe behavior. +- [ ] Paired performance evidence shows no repeatable regression at comparable resources, with fingerprints proving equivalent work. Unmeasured platforms and inconclusive results remain disclosed limitations. +- [ ] Required tests/typecheck pass, graph impact/change analysis is complete, and the LFG review verdict and remaining findings are reported. No unrelated edits, push or PR publication. From c358007397a764a5e990207f74be139d5a22131c Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 09:51:32 +0000 Subject: [PATCH 02/67] perf(bench): measure resumable storage primitives --- gitnexus/bench/analyze-resume/README.md | 25 ++ .../bench/analyze-resume/native-baseline.json | 121 ++++++++++ .../analyze-resume/native-primitives.mjs | 220 ++++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 gitnexus/bench/analyze-resume/README.md create mode 100644 gitnexus/bench/analyze-resume/native-baseline.json create mode 100644 gitnexus/bench/analyze-resume/native-primitives.mjs diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md new file mode 100644 index 000000000..6070c9519 --- /dev/null +++ b/gitnexus/bench/analyze-resume/README.md @@ -0,0 +1,25 @@ +# Resumable analyze measurements + +Run the native feasibility check from `gitnexus/`: + +```sh +node --expose-gc bench/analyze-resume/native-primitives.mjs --check +``` + +The probe alternates baseline and candidate order, checks identical content and +vector coverage, verifies primary-key lookup in the native query plan, and opens +the published graph in a separate process while a vector writer remains open. +The write candidate includes active-intent fsync, a transaction per unit, native +checkpointing, and completed-cursor fsync. `--check` rejects a slower candidate +median for either reader or writer. + +`native-baseline.json` records the initial Linux/Node 22 samples. The bounded +reader retained at most 1 MiB of content per batch for 10,000 nodes containing +40.96 MB of content. It still holds a small key array to isolate native fetch +cost; production manifest memory must be tested separately. The writer probe +uses 512 vectors with 256 dimensions and 64 vectors per durability unit. + +These measurements establish native feasibility. They do not establish +end-to-end analyze throughput, crash recovery, model throughput, or total RSS +bounds. Run the application regression matrix before making those claims. +Windows handle behavior and local-model execution also need their own runs. diff --git a/gitnexus/bench/analyze-resume/native-baseline.json b/gitnexus/bench/analyze-resume/native-baseline.json new file mode 100644 index 000000000..8e4a439ee --- /dev/null +++ b/gitnexus/bench/analyze-resume/native-baseline.json @@ -0,0 +1,121 @@ +{ + "node": "v22.23.3", + "platform": "linux", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "nodes": 10000, + "unit": 256, + "vectorNodes": 512, + "dimensions": 256, + "samples": [ + { + "phase": "read", + "trial": 0, + "mode": "baseline", + "ms": 560.4716119999999, + "maximumContentBytes": 40960000 + }, + { + "phase": "read", + "trial": 0, + "mode": "bounded", + "ms": 84.59907299999998, + "maximumContentBytes": 1048576 + }, + { + "phase": "read", + "trial": 1, + "mode": "bounded", + "ms": 75.56121000000007, + "maximumContentBytes": 1048576 + }, + { + "phase": "read", + "trial": 1, + "mode": "baseline", + "ms": 601.821259, + "maximumContentBytes": 40960000 + }, + { + "phase": "read", + "trial": 2, + "mode": "baseline", + "ms": 597.1018180000001, + "maximumContentBytes": 40960000 + }, + { + "phase": "read", + "trial": 2, + "mode": "bounded", + "ms": 76.60035700000026, + "maximumContentBytes": 1048576 + }, + { + "phase": "read", + "trial": 3, + "mode": "bounded", + "ms": 77.93451700000014, + "maximumContentBytes": 1048576 + }, + { + "phase": "read", + "trial": 3, + "mode": "baseline", + "ms": 605.307159, + "maximumContentBytes": 40960000 + }, + { + "phase": "read", + "trial": 4, + "mode": "baseline", + "ms": 566.7196509999999, + "maximumContentBytes": 40960000 + }, + { + "phase": "read", + "trial": 4, + "mode": "bounded", + "ms": 68.64343800000006, + "maximumContentBytes": 1048576 + }, + { + "phase": "write", + "trial": 0, + "mode": "baseline", + "ms": 3262.1321529999996 + }, + { + "phase": "write", + "trial": 0, + "mode": "bounded", + "ms": 683.4758389999997 + }, + { + "phase": "write", + "trial": 1, + "mode": "bounded", + "ms": 692.3224590000009 + }, + { + "phase": "write", + "trial": 1, + "mode": "baseline", + "ms": 3120.299986 + }, + { + "phase": "write", + "trial": 2, + "mode": "baseline", + "ms": 2709.020754000001 + }, + { + "phase": "write", + "trial": 2, + "mode": "bounded", + "ms": 642.1225830000003 + } + ], + "ratios": { + "read": 0.12828692643504938, + "write": 0.21904170819042518 + } +} diff --git a/gitnexus/bench/analyze-resume/native-primitives.mjs b/gitnexus/bench/analyze-resume/native-primitives.mjs new file mode 100644 index 000000000..1ac5512ba --- /dev/null +++ b/gitnexus/bench/analyze-resume/native-primitives.mjs @@ -0,0 +1,220 @@ +#!/usr/bin/env node +// Native feasibility probe, not an end-to-end analyze performance claim. +// node --expose-gc bench/analyze-resume/native-primitives.mjs [--check] +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const require = createRequire(import.meta.url); +const { Database, Connection } = require('@ladybugdb/core'); +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-analyze-resume-')); +const nodes = 10_000; +const content = 'x'.repeat(4096); +const unit = 256; +const vectorNodes = 512; +const dimensions = 256; +const samples = []; + +async function query(connection, text, parameters) { + const result = parameters + ? await connection.execute(await connection.prepare(text), parameters) + : await connection.query(text); + try { + return await result.getAll(); + } finally { + result.close(); + } +} + +async function durableJson(file, value) { + const handle = await fs.open(`${file}.tmp`, 'w'); + try { + await handle.writeFile(JSON.stringify(value)); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(`${file}.tmp`, file); + // Directory fsync is unavailable on Windows; file fsync still runs there. + if (process.platform !== 'win32') { + const directory = await fs.open(path.dirname(file), 'r'); + try { + await directory.sync(); + } finally { + await directory.close(); + } + } +} + +const graphPath = path.join(root, 'graph'); +let graph; +let graphConnection; +try { + graph = new Database(graphPath, 128 * 1024 * 1024); + await graph.init(); + graphConnection = new Connection(graph, 2); + await query(graphConnection, 'CREATE NODE TABLE N(id STRING, content STRING, PRIMARY KEY(id))'); + await query( + graphConnection, + `UNWIND range(0, ${nodes - 1}) AS i CREATE (:N {id: cast(i AS STRING), content: $content})`, + { content }, + ); + await query(graphConnection, 'CHECKPOINT'); + await graphConnection.close(); + await graph.close(); + graph = new Database(graphPath, 128 * 1024 * 1024, true, true); + await graph.init(); + graphConnection = new Connection(graph, 2); + const fetchText = 'UNWIND $ids AS id MATCH (n:N {id:id}) RETURN n.id AS id, n.content AS content'; + const fetch = await graphConnection.prepare(fetchText); + const plan = await query(graphConnection, `EXPLAIN ${fetchText}`, { ids: ['0'] }); + assert.match(JSON.stringify(plan), /PRIMARY_KEY_LOOKUP/); + + for (let trial = 0; trial < 5; trial++) { + for (const mode of trial % 2 ? ['bounded', 'baseline'] : ['baseline', 'bounded']) { + global.gc?.(); + const start = performance.now(); + let bytes = 0; + let maximumContentBytes = 0; + if (mode === 'baseline') { + const rows = await query( + graphConnection, + 'MATCH (n:N) RETURN n.id AS id, n.content AS content', + ); + for (const row of rows) bytes += Buffer.byteLength(row.content); + maximumContentBytes = bytes; + } else { + // This probe isolates native fetch cost. Production must persist keys + // in its manifest; this small key array is not a bounded manifest. + const keys = []; + const result = await graphConnection.query('MATCH (n:N) RETURN n.id AS id ORDER BY n.id'); + try { + while (result.hasNext()) keys.push(result.getNextSync().id); + } finally { + result.close(); + } + for (let offset = 0; offset < keys.length; offset += unit) { + const rows = await graphConnection.execute(fetch, { + ids: keys.slice(offset, offset + unit), + }); + let batchBytes = 0; + try { + while (rows.hasNext()) batchBytes += Buffer.byteLength(rows.getNextSync().content); + } finally { + rows.close(); + } + bytes += batchBytes; + maximumContentBytes = Math.max(maximumContentBytes, batchBytes); + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok(maximumContentBytes <= unit * Buffer.byteLength(content)); + } + assert.equal(bytes, nodes * Buffer.byteLength(content)); + samples.push({ + phase: 'read', + trial, + mode, + ms: performance.now() - start, + maximumContentBytes, + }); + } + } + + for (let trial = 0; trial < 3; trial++) { + for (const mode of trial % 2 ? ['bounded', 'baseline'] : ['baseline', 'bounded']) { + const vectors = new Database(path.join(root, `vectors-${trial}-${mode}`), 64 * 1024 * 1024); + await vectors.init(); + const writer = new Connection(vectors, 2); + try { + await query( + writer, + `CREATE NODE TABLE E(id STRING, embedding FLOAT[${dimensions}], PRIMARY KEY(id))`, + ); + // A fresh process must be able to read the published graph while the + // vector writer is open. Same-process handle sharing cannot prove this. + const child = execFileSync( + process.execPath, + [ + '-e', + ` + const {Database,Connection}=require(${JSON.stringify(require.resolve('@ladybugdb/core'))}); + (async()=>{const db=new Database(${JSON.stringify(graphPath)},67108864,true,true); + await db.init();const c=new Connection(db,2); + const r=await c.query('MATCH (n:N) RETURN count(n) AS count'); + require('node:fs').writeSync(1,JSON.stringify(await r.getAll()));r.close(); + await c.close();await db.close();})().catch(e=>{console.error(e);process.exitCode=1}); + `, + ], + { encoding: 'utf8', timeout: 30_000 }, + ); + assert.equal(JSON.parse(child)[0].count, nodes); + const insert = await writer.prepare('CREATE (:E {id:$id, embedding:$embedding})'); + const embedding = Array(dimensions).fill(0.125); + const start = performance.now(); + for (let offset = 0; offset < vectorNodes; offset += 64) { + if (mode === 'bounded') { + await durableJson(path.join(root, 'job.json'), { + completed: offset, + active: offset / 64, + }); + await query(writer, 'BEGIN TRANSACTION'); + } + for (let i = offset; i < offset + 64; i++) { + const result = await writer.execute(insert, { id: String(i), embedding }); + result.close(); + } + if (mode === 'bounded') { + await query(writer, 'COMMIT'); + await query(writer, 'CHECKPOINT'); + await durableJson(path.join(root, 'job.json'), { completed: offset + 64 }); + } + } + await query(writer, 'CHECKPOINT'); + samples.push({ phase: 'write', trial, mode, ms: performance.now() - start }); + assert.equal( + (await query(writer, 'MATCH (n:E) RETURN count(n) AS count'))[0].count, + vectorNodes, + ); + } finally { + await writer.close(); + await vectors.close(); + } + } + } + const median = (values) => values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]; + const ratios = Object.fromEntries( + ['read', 'write'].map((phase) => [ + phase, + median(samples.filter((s) => s.phase === phase && s.mode === 'bounded').map((s) => s.ms)) / + median(samples.filter((s) => s.phase === phase && s.mode === 'baseline').map((s) => s.ms)), + ]), + ); + console.log( + JSON.stringify( + { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + nodes, + unit, + vectorNodes, + dimensions, + samples, + ratios, + }, + null, + 2, + ), + ); + if (process.argv.includes('--check')) { + for (const [phase, ratio] of Object.entries(ratios)) + assert.ok(ratio <= 1, `${phase} regressed: ${ratio}`); + } +} finally { + await graphConnection?.close(); + await graph?.close(); + await fs.rm(root, { recursive: true, force: true }); +} From 6c12d89dd79b73060a3746f42f73fab3e1d40086 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 10:16:57 +0000 Subject: [PATCH 03/67] feat(embeddings): bound and persist work selection --- gitnexus/src/core/analysis-resource-budget.ts | 207 ++++++++++ .../embeddings/embedding-work-manifest.ts | 385 ++++++++++++++++++ gitnexus/src/storage/embedding-job.ts | 95 +++++ .../unit/analysis-resource-budget.test.ts | 105 +++++ .../test/unit/embedding-work-manifest.test.ts | 261 ++++++++++++ 5 files changed, 1053 insertions(+) create mode 100644 gitnexus/src/core/analysis-resource-budget.ts create mode 100644 gitnexus/src/core/embeddings/embedding-work-manifest.ts create mode 100644 gitnexus/src/storage/embedding-job.ts create mode 100644 gitnexus/test/unit/analysis-resource-budget.test.ts create mode 100644 gitnexus/test/unit/embedding-work-manifest.test.ts diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts new file mode 100644 index 000000000..f796b4fb2 --- /dev/null +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -0,0 +1,207 @@ +/** Pure admission policy. Reservations are not an OS-enforced RSS limit. */ +const MIB = 1024 * 1024; +const GRAPH_POOL_MB = 256; +const VECTOR_POOL_MB = 64; +const MIN_MAIN_HEAP_MB = 384; +const MIN_WORKER_HEAP_MB = 512; +const WORKER_NATIVE_MB = 128; + +export interface AnalysisResourceRequest { + /** Effective process/container RAM, before reserving host headroom. */ + effectiveMemoryBytes: number; + availableCpus: number; + /** Explicit total reservation ceiling; independent of the heap-only flag. */ + totalMb?: number; + mainHeapMb?: number; + workers?: number; + workerHeapMb?: number; + /** Existing automatic worker policy; explicit workers still wins. */ + preferredWorkers?: number; + /** Resume/sync can admit embeddings without reserving parser workers. */ + includeParsing?: boolean; + graphPoolBytes?: number; + /** Estimated peak model/sidecar allocation; zero for an HTTP provider. */ + modelMemoryMb: number; +} + +export interface AnalysisResourceBudget { + readonly totalBytes: number; + readonly automatic: boolean; + readonly headroomBytes: number; + readonly mainHeapMb: number; + readonly parse: { + readonly workers: number; + readonly workerHeapMb: number; + readonly workerNativeBytes: number; + readonly graphPoolBytes: number; + readonly reservedBytes: number; + }; + readonly embedding: { + readonly graphPoolBytes: number; + readonly vectorPoolBytes: number; + readonly modelBytes: number; + /** Combined source, expanded text, encoded request, vectors and prefetch. */ + readonly queueBytes: number; + readonly nativeThreads: number; + readonly reservedBytes: number; + }; +} + +function integer(value: number, label: string, minimum = 1): number { + if (!Number.isSafeInteger(value) || value < minimum) { + throw new Error(`${label} must be a safe integer of at least ${minimum}`); + } + return value; +} + +function bytes(mb: number, label: string): number { + return integer(integer(mb, label, 0) * MIB, `${label} in bytes`, 0); +} + +/** Resolve before opening a writable database or starting parser/model work. */ +export function resolveAnalysisResourceBudget( + request: AnalysisResourceRequest, +): AnalysisResourceBudget { + const memory = integer(request.effectiveMemoryBytes, 'effectiveMemoryBytes'); + const cpus = integer(request.availableCpus, 'availableCpus'); + const availableMb = Math.floor(memory / MIB); + const totalMb = + request.totalMb === undefined + ? Math.floor(availableMb * 0.75) + : integer(request.totalMb, 'resource budget'); + if (totalMb > availableMb) { + throw new Error( + `Resource budget ${totalMb} MiB exceeds effective process memory ${availableMb} MiB`, + ); + } + const totalBytes = bytes(totalMb, 'resource budget'); + const headroomMb = Math.max(128, Math.ceil(totalMb * 0.12)); + const graphPoolBytes = + request.graphPoolBytes === undefined + ? bytes(Math.max(GRAPH_POOL_MB, Math.min(4096, Math.floor(totalMb * 0.15))), 'graph pool') + : integer(request.graphPoolBytes, 'graph pool bytes'); + // Zero is the native engine's unbounded override, never a bounded allocation. + const graphPoolMb = Math.ceil(graphPoolBytes / MIB); + const modelMb = integer(request.modelMemoryMb, 'model reservation', 0); + const queueMb = Math.max(16, Math.min(128, Math.floor(totalMb / 32))); + const embeddingFixedMb = headroomMb + graphPoolMb + VECTOR_POOL_MB + modelMb + queueMb; + const includeParsing = request.includeParsing !== false; + const parseFixedMb = + headroomMb + graphPoolMb + (includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0); + const maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); + const mainHeapMb = + request.mainHeapMb === undefined + ? Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) + : integer(request.mainHeapMb, 'main heap reservation'); + if (mainHeapMb < MIN_MAIN_HEAP_MB || mainHeapMb > maximumMainMb) { + const minimum = + Math.max(MIN_MAIN_HEAP_MB, request.mainHeapMb ?? 0) + + Math.max(embeddingFixedMb, parseFixedMb); + throw new Error( + `Resource budget ${totalMb} MiB cannot admit parser/model work; requires at least ${minimum} MiB for these allocations`, + ); + } + + const cpuWorkers = Math.max(1, cpus - 1); + const desiredWorkers = + request.workers === undefined + ? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers) + : integer(request.workers, 'parser workers'); + if (includeParsing && desiredWorkers > cpuWorkers) { + throw new Error( + `Requested ${desiredWorkers} parser workers exceeds the ${cpuWorkers}-worker CPU allowance`, + ); + } + const workerBudgetMb = totalMb - headroomMb - graphPoolMb - mainHeapMb; + const explicitWorkerHeapMb = + request.workerHeapMb === undefined + ? undefined + : integer(request.workerHeapMb, 'worker heap reservation'); + if ( + includeParsing && + explicitWorkerHeapMb !== undefined && + explicitWorkerHeapMb < MIN_WORKER_HEAP_MB + ) { + throw new Error(`Worker heap reservation must be at least ${MIN_WORKER_HEAP_MB} MiB`); + } + const workers = !includeParsing + ? 0 + : request.workers === undefined + ? Math.min( + desiredWorkers, + Math.floor( + workerBudgetMb / ((explicitWorkerHeapMb ?? MIN_WORKER_HEAP_MB) + WORKER_NATIVE_MB), + ), + ) + : desiredWorkers; + const workerHeapMb = !includeParsing + ? 0 + : (explicitWorkerHeapMb ?? + Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB)); + const parseReservedMb = + headroomMb + graphPoolMb + mainHeapMb + workers * (workerHeapMb + WORKER_NATIVE_MB); + if ( + (includeParsing && (workers < 1 || workerHeapMb < MIN_WORKER_HEAP_MB)) || + parseReservedMb > totalMb + ) { + throw new Error( + `Parser workers do not fit resource budget ${totalMb} MiB; reduce workers or explicit heap reservations`, + ); + } + const embeddingReservedMb = mainHeapMb + embeddingFixedMb; + return Object.freeze({ + totalBytes, + automatic: request.totalMb === undefined, + headroomBytes: bytes(headroomMb, 'headroom'), + mainHeapMb, + parse: Object.freeze({ + workers, + workerHeapMb, + workerNativeBytes: bytes(WORKER_NATIVE_MB, 'worker native reservation'), + graphPoolBytes, + reservedBytes: bytes(parseReservedMb, 'parse reservation'), + }), + embedding: Object.freeze({ + graphPoolBytes, + vectorPoolBytes: bytes(VECTOR_POOL_MB, 'vector pool'), + modelBytes: bytes(modelMb, 'model reservation'), + queueBytes: bytes(queueMb, 'embedding queues'), + nativeThreads: Math.min(4, cpus), + reservedBytes: bytes(embeddingReservedMb, 'embedding reservation'), + }), + }); +} + +/** Account retained payloads before admission, including a prefetched unit. */ +export class EmbeddingByteBudget { + private retained = 0; + private peak = 0; + + constructor(readonly limit: number) { + integer(limit, 'embedding byte budget'); + } + + get usedBytes(): number { + return this.retained; + } + get peakBytes(): number { + return this.peak; + } + + reserve(size: number): () => void { + integer(size, 'retained payload bytes', 0); + if (size > this.limit - this.retained) { + throw new Error( + `Embedding payload requires ${size} bytes with ${this.limit - this.retained} bytes available`, + ); + } + this.retained += size; + this.peak = Math.max(this.peak, this.retained); + let released = false; + return () => { + if (released) throw new Error('Embedding byte reservation released twice'); + released = true; + this.retained -= size; + }; + } +} diff --git a/gitnexus/src/core/embeddings/embedding-work-manifest.ts b/gitnexus/src/core/embeddings/embedding-work-manifest.ts new file mode 100644 index 000000000..2de60655e --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-work-manifest.ts @@ -0,0 +1,385 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import fs, { type FileHandle } from 'node:fs/promises'; +import path from 'node:path'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; +import { EMBEDDABLE_LABELS, type EmbeddableNode } from './types.js'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from '../../storage/embedding-job.js'; + +// Persisted unit identity must not depend on a retry's memory/batch settings. +const UNIT_NODES = 64; +const UNIT_SOURCE_BYTES = 1024 * 1024; +const MAX_LINE_BYTES = 1024 * 1024; +const IO_BYTES = 64 * 1024; +const LABELS = new Set([...EMBEDDABLE_LABELS, 'File']); + +export interface EmbeddingWorkKey { + id: string; + contentBytes: number; + /** Upper bound for serialized persisted node metadata, including its ID. */ + metadataBytes: number; +} + +export interface EmbeddingManifestSource { + /** Key-only scan in UTF-8 ID order. File scans yield only eligible text files. */ + scan(label: string): AsyncIterable; + /** Bounded primary-key fetch against the same immutable graph generation. */ + fetch(label: string, ids: readonly string[]): Promise; +} + +export interface EmbeddingWorkUnit { + ordinal: number; + label: string; + nodes: EmbeddingWorkKey[]; + /** Source strings plus persisted metadata; excludes later text/vector queues. */ + retainedBytes: number; +} + +export interface EmbeddingManifestReceipt { + version: 1; + sourceIdentity: string; + digest: string; + bytes: number; + totalNodes: number; + totalUnits: number; + maximumUnitBytes: number; +} + +export interface EmbeddingManifestCursor { + unit: number; + offset: number; +} + +function count(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} + +function keyBytes(key: EmbeddingWorkKey): number { + return 2 * (key.contentBytes + key.metadataBytes); +} + +function validateKey(value: unknown): asserts value is EmbeddingWorkKey { + if (!value || typeof value !== 'object' || !('id' in value)) + throw new Error('Malformed embedding manifest key'); + assertValidGraphNodeId(value.id, 'Embedding manifest'); + if ( + Buffer.from(value.id).toString('utf8') !== value.id || + !('contentBytes' in value) || + !count(value.contentBytes) || + !('metadataBytes' in value) || + !count(value.metadataBytes) || + value.metadataBytes < Buffer.byteLength(value.id) + ) { + throw new Error('Invalid embedding manifest key bounds'); + } + if (!Number.isSafeInteger(2 * (value.contentBytes + value.metadataBytes))) + throw new Error('Embedding key size exceeds safe accounting'); +} + +class BufferedLines { + private pending: string[] = []; + private size = 0; + constructor(private readonly handle: FileHandle) {} + async write(line: string): Promise { + const size = Buffer.byteLength(line); + if (size > MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + this.pending.push(line); + this.size += size; + if (this.size >= IO_BYTES) await this.flush(); + } + async flush(): Promise { + if (!this.size) return; + await this.handle.writeFile(this.pending.join('')); + this.pending = []; + this.size = 0; + } +} + +/** Framing stays bounded even for a corrupt file without a newline. */ +async function* lines(file: string, start = 0): AsyncGenerator<{ text: string; end: number }> { + if (!(await fs.lstat(file)).isFile()) + throw new Error('Embedding manifest artifact must be a regular file'); + const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + if (!(await handle.stat()).isFile()) + throw new Error('Embedding manifest artifact must be a regular file'); + const buffer = Buffer.alloc(IO_BYTES); + let pending = Buffer.alloc(0); + let position = start; + let lineStart = start; + for (;;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (!bytesRead) break; + position += bytesRead; + const data = pending.length + ? Buffer.concat([pending, buffer.subarray(0, bytesRead)]) + : buffer.subarray(0, bytesRead); + let offset = 0; + for (;;) { + const end = data.indexOf(10, offset); + if (end < 0) break; + if (end - offset + 1 > MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + lineStart += end - offset + 1; + yield { + text: new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)), + end: lineStart, + }; + offset = end + 1; + } + pending = Buffer.from(data.subarray(offset)); + if (pending.length >= MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + } + if (pending.length) throw new Error('Embedding manifest has an incomplete trailing record'); + } finally { + await handle.close(); + } +} + +/** Merge label-key spools to detect cross-label duplicates without a corpus Set. */ +async function verifyUniqueKeys(files: string[]): Promise { + const readers = files.map((file) => lines(file)); + const heads: Array<{ id: string; bytes: Buffer } | undefined> = []; + const head = (text: string): { id: string; bytes: Buffer } => { + const { id }: { id: string } = JSON.parse(text); + return { id, bytes: Buffer.from(id) }; + }; + try { + for (const reader of readers) { + const next = await reader.next(); + heads.push(next.done ? undefined : head(next.value.text)); + } + let previous: string | undefined; + for (;;) { + let selected = -1; + for (let i = 0; i < heads.length; i++) { + if ( + heads[i] && + (selected < 0 || Buffer.compare(heads[i]!.bytes, heads[selected]!.bytes) < 0) + ) + selected = i; + } + if (selected < 0) break; + const id = heads[selected]!.id; + if (id === previous) + throw new Error( + 'Embedding read: duplicate graph node identity; rebuild with analyze --force', + ); + previous = id; + const next = await readers[selected].next(); + heads[selected] = next.done ? undefined : head(next.value.text); + } + } finally { + await Promise.all(readers.map((reader) => reader.return(undefined))); + } +} + +/** Preflight and publish the complete selection before any provider/write work. */ +export async function createEmbeddingWorkManifest( + directory: string, + sourceIdentity: string, + source: EmbeddingManifestSource, +): Promise { + if (!sourceIdentity || Buffer.byteLength(sourceIdentity) > 512) + throw new Error('Invalid manifest source identity'); + const canonical = await embeddingJobDirectory(directory); + const pendingPath = path.join(canonical, 'manifest.pending'); + const handle = await fs.open(pendingPath, 'wx', 0o600); + const output = new BufferedLines(handle); + const hash = createHash('sha256'); + const keyFiles: string[] = []; + let totalNodes = 0; + let totalUnits = 0; + let maximumUnitBytes = 0; + let bytes = 0; + try { + const scanLabel = async (label: string): Promise => { + const keyPath = path.join(canonical, `keys-${keyFiles.length}.jsonl`); + const keysHandle = await fs.open(keyPath, 'wx', 0o600); + keyFiles.push(keyPath); + const keyOutput = new BufferedLines(keysHandle); + let previous: Buffer | undefined; + let nodes: EmbeddingWorkKey[] = []; + let retainedBytes = 0; + let recordBytes = 0; + const flush = async (): Promise => { + if (!nodes.length) return; + const unit: EmbeddingWorkUnit = { ordinal: totalUnits++, label, nodes, retainedBytes }; + const line = `${JSON.stringify(unit)}\n`; + await output.write(line); + hash.update(line); + bytes += Buffer.byteLength(line); + maximumUnitBytes = Math.max(maximumUnitBytes, retainedBytes); + nodes = []; + retainedBytes = 0; + recordBytes = 0; + }; + try { + for await (const key of source.scan(label)) { + validateKey(key); + const id = Buffer.from(key.id); + if (previous && Buffer.compare(previous, id) >= 0) + throw new Error( + `Embedding ${label} key scan is unordered or contains duplicate identities`, + ); + previous = id; + const serialized = `${JSON.stringify(key)}\n`; + const size = Buffer.byteLength(serialized); + if (size + 1024 > MAX_LINE_BYTES) + throw new Error('Embedding identity exceeds the manifest record budget'); + if ( + nodes.length && + (nodes.length === UNIT_NODES || + retainedBytes + keyBytes(key) > UNIT_SOURCE_BYTES || + recordBytes + size + 1024 > MAX_LINE_BYTES) + ) + await flush(); + await keyOutput.write(serialized); + nodes.push({ + id: key.id, + contentBytes: key.contentBytes, + metadataBytes: key.metadataBytes, + }); + retainedBytes += keyBytes(key); + recordBytes += size; + totalNodes++; + } + await flush(); + await keyOutput.flush(); + } finally { + await keysHandle.close(); + } + }; + for (const label of EMBEDDABLE_LABELS) await scanLabel(label); + if (totalNodes === 0) await scanLabel('File'); + await verifyUniqueKeys(keyFiles); + await output.flush(); + await handle.sync(); + } finally { + await handle.close(); + } + const receipt: EmbeddingManifestReceipt = { + version: 1, + sourceIdentity, + digest: hash.digest('hex'), + bytes, + totalNodes, + totalUnits, + maximumUnitBytes, + }; + await fs.rename(pendingPath, path.join(canonical, 'manifest.jsonl')); + await syncEmbeddingJobDirectory(canonical); + await writeEmbeddingJobRecord(canonical, 'manifest.json', receipt); + await Promise.all(keyFiles.map((file) => fs.unlink(file))); + return receipt; +} + +/** Validate the durable selection without re-reading graph content or keys. */ +export async function loadEmbeddingWorkManifest( + directory: string, + sourceIdentity: string, +): Promise { + const value = await readEmbeddingJobRecord(directory, 'manifest.json'); + if (!value || typeof value !== 'object') throw new Error('Malformed embedding manifest receipt'); + const receipt = value as EmbeddingManifestReceipt; + if ( + receipt.version !== 1 || + receipt.sourceIdentity !== sourceIdentity || + !/^[0-9a-f]{64}$/.test(receipt.digest) || + !count(receipt.bytes) || + !count(receipt.totalNodes) || + !count(receipt.totalUnits) || + !count(receipt.maximumUnitBytes) + ) { + throw new Error('Embedding manifest identity or version mismatch; artifacts retained'); + } + const canonical = await embeddingJobDirectory(directory); + const hash = createHash('sha256'); + let bytes = 0; + for await (const line of lines(path.join(canonical, 'manifest.jsonl'))) { + hash.update(`${line.text}\n`); + bytes = line.end; + if (bytes > receipt.bytes) throw new Error('Embedding manifest size mismatch'); + } + if (bytes !== receipt.bytes || hash.digest('hex') !== receipt.digest) + throw new Error('Embedding manifest checksum mismatch; artifacts retained'); + return receipt; +} + +export async function* readEmbeddingWorkUnits( + directory: string, + receipt: EmbeddingManifestReceipt, + cursor: EmbeddingManifestCursor = { unit: 0, offset: 0 }, +): AsyncGenerator<{ unit: EmbeddingWorkUnit; next: EmbeddingManifestCursor }> { + if ( + !count(cursor.unit) || + cursor.unit > receipt.totalUnits || + !count(cursor.offset) || + cursor.offset > receipt.bytes + ) + throw new Error('Invalid embedding manifest cursor'); + const canonical = await embeddingJobDirectory(directory); + let ordinal = cursor.unit; + for await (const line of lines(path.join(canonical, 'manifest.jsonl'), cursor.offset)) { + const unit: EmbeddingWorkUnit = JSON.parse(line.text); + if ( + unit.ordinal !== ordinal || + !LABELS.has(unit.label) || + !Array.isArray(unit.nodes) || + !unit.nodes.length || + unit.nodes.length > UNIT_NODES + ) + throw new Error('Invalid embedding work unit'); + unit.nodes.forEach(validateKey); + if ( + unit.retainedBytes !== unit.nodes.reduce((sum, key) => sum + keyBytes(key), 0) || + unit.retainedBytes > receipt.maximumUnitBytes + ) + throw new Error('Invalid embedding unit byte accounting'); + ordinal++; + yield { unit, next: { unit: ordinal, offset: line.end } }; + } + if (ordinal !== receipt.totalUnits) + throw new Error('Embedding manifest cursor does not cover the remaining units'); +} + +export async function fetchEmbeddingWorkUnit( + source: EmbeddingManifestSource, + unit: EmbeddingWorkUnit, + budget: EmbeddingByteBudget, +): Promise<{ nodes: EmbeddableNode[]; release: () => void }> { + const release = budget.reserve(unit.retainedBytes); + try { + const rows = await source.fetch( + unit.label, + unit.nodes.map((node) => node.id), + ); + const byId = new Map(rows.map((row) => [row.id, row])); + if (rows.length !== unit.nodes.length || byId.size !== rows.length) + throw new Error('Embedding source returned missing or duplicate nodes'); + const nodes = unit.nodes.map((key) => { + const node = byId.get(key.id); + if ( + !node || + node.label !== unit.label || + typeof node.content !== 'string' || + Buffer.byteLength(node.content) !== key.contentBytes || + Buffer.byteLength(JSON.stringify({ ...node, content: undefined })) > key.metadataBytes + ) + throw new Error('Embedding graph changed after manifest publication'); + return node; + }); + return { nodes, release }; + } catch (error) { + release(); + throw error; + } +} diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts new file mode 100644 index 000000000..4f6ff3d2a --- /dev/null +++ b/gitnexus/src/storage/embedding-job.ts @@ -0,0 +1,95 @@ +/** Filesystem-only embedding receipts; importing this never loads a model or DB. */ +import { createHash, randomUUID } from 'node:crypto'; +import { constants } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +const MAX_RECORD_BYTES = 32 * 1024; +const RECORD_NAMES = new Set(['manifest.json', 'job.json']); + +export async function embeddingJobDirectory(directory: string): Promise { + const stat = await fs.lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Embedding job directory must be an owned directory'); + return fs.realpath(directory); +} + +/** Flush directory entries where the platform supports opening directories. */ +export async function syncEmbeddingJobDirectory(directory: string): Promise { + if (process.platform === 'win32') return; + const handle = await fs.open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0)); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function writeEmbeddingJobRecord( + directory: string, + name: string, + payload: unknown, +): Promise { + if (!RECORD_NAMES.has(name)) throw new Error('Invalid embedding receipt name'); + const canonical = await embeddingJobDirectory(directory); + const serialized = JSON.stringify(payload); + if (serialized === undefined) throw new Error('Embedding receipt payload is not JSON'); + const checksum = createHash('sha256').update(serialized).digest('hex'); + const bytes = Buffer.from(JSON.stringify({ checksum, payload }), 'utf8'); + if (bytes.byteLength > MAX_RECORD_BYTES) + throw new Error('Embedding receipt exceeds its fixed size limit'); + const temporary = path.join(canonical, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(canonical, name)); + await syncEmbeddingJobDirectory(canonical); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +/** Bounded read; malformed or untrusted receipts never become an empty job. */ +export async function readEmbeddingJobRecord(directory: string, name: string): Promise { + if (!RECORD_NAMES.has(name)) throw new Error('Invalid embedding receipt name'); + const canonical = await embeddingJobDirectory(directory); + const target = path.join(canonical, name); + if (!(await fs.lstat(target)).isFile()) + throw new Error('Embedding receipt must be a regular file'); + const handle = await fs.open(target, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + if (!(await handle.stat()).isFile()) + throw new Error('Embedding receipt must be a regular file'); + const buffer = Buffer.alloc(MAX_RECORD_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > MAX_RECORD_BYTES) + throw new Error('Embedding receipt exceeds its fixed size limit'); + const parsed: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)), + ); + if ( + !parsed || + typeof parsed !== 'object' || + !('checksum' in parsed) || + !('payload' in parsed) + ) { + throw new Error('Malformed embedding receipt'); + } + const checksum = createHash('sha256').update(JSON.stringify(parsed.payload)).digest('hex'); + if (parsed.checksum !== checksum) + throw new Error('Embedding receipt checksum mismatch; artifacts retained'); + return parsed.payload; + } finally { + await handle.close(); + } +} diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts new file mode 100644 index 000000000..4c5f5c7d3 --- /dev/null +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { + EmbeddingByteBudget, + resolveAnalysisResourceBudget, +} from '../../src/core/analysis-resource-budget.js'; + +const MIB = 1024 * 1024; +const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; + +describe('analysis resource admission', () => { + it('bounds every phase separately and reuses parser capacity for embedding', () => { + for (const memoryMb of [2048, 4096, 8192, 16384, 32768]) { + for (const availableCpus of [1, 2, 4, 24]) { + const budget = resolveAnalysisResourceBudget({ + ...request, + effectiveMemoryBytes: memoryMb * MIB, + availableCpus, + modelMemoryMb: 0, + }); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.totalBytes).toBeLessThan(memoryMb * MIB); + expect(budget.parse.workers).toBeLessThanOrEqual(Math.max(1, availableCpus - 1)); + expect(budget.embedding.nativeThreads).toBeLessThanOrEqual(availableCpus); + } + } + }); + + it('preserves explicit main-heap semantics and preferred concurrency when they fit', () => { + const budget = resolveAnalysisResourceBudget({ + ...request, + totalMb: 24_576, + mainHeapMb: 8192, + preferredWorkers: 6, + }); + expect(budget.mainHeapMb).toBe(8192); + expect(budget.parse.workers).toBe(6); + expect(budget.automatic).toBe(false); + expect(Object.isFrozen(budget)).toBe(true); + expect(Object.isFrozen(budget.parse)).toBe(true); + expect(Object.isFrozen(budget.embedding)).toBe(true); + }); + + it('admits an HTTP embedding-only resume below the parser minimum', () => { + const input = { ...request, totalMb: 1024, modelMemoryMb: 0 }; + expect(() => resolveAnalysisResourceBudget(input)).toThrow('cannot admit'); + const budget = resolveAnalysisResourceBudget({ ...input, includeParsing: false }); + expect(budget.parse.workers).toBe(0); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(1024 * MIB); + }); + + it('rejects impossible explicit combinations before work starts', () => { + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 128 })).toThrow( + 'cannot admit', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 65536 })).toThrow( + 'exceeds effective', + ); + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 4096, mainHeapMb: 8192 }), + ).toThrow('cannot admit'); + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 4, workerHeapMb: 4096 }), + ).toThrow('do not fit'); + expect(() => resolveAnalysisResourceBudget({ ...request, workers: 24 })).toThrow( + 'CPU allowance', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: 0 })).toThrow( + 'graph pool bytes', + ); + }); + + it.each([NaN, Infinity, -1, 0, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects malformed total budget %s', + (totalMb) => { + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb })).toThrow(); + }, + ); +}); + +describe('embedding payload accounting', () => { + it('accounts an active payload and prefetch together, with a stable peak', () => { + const budget = new EmbeddingByteBudget(1024); + for (let i = 0; i < 100_000; i++) { + const active = budget.reserve(512); + const prefetched = budget.reserve(512); + expect(budget.usedBytes).toBe(1024); + prefetched(); + active(); + } + expect(budget.usedBytes).toBe(0); + expect(budget.peakBytes).toBe(1024); + }); + + it('rejects overflow without leaking capacity and detects double release', () => { + const budget = new EmbeddingByteBudget(1024); + const release = budget.reserve(768); + expect(() => budget.reserve(257)).toThrow('256 bytes available'); + expect(budget.usedBytes).toBe(768); + release(); + expect(() => release()).toThrow('released twice'); + expect(budget.usedBytes).toBe(0); + expect(() => budget.reserve(-1)).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/embedding-work-manifest.test.ts b/gitnexus/test/unit/embedding-work-manifest.test.ts new file mode 100644 index 000000000..c42c7b1b9 --- /dev/null +++ b/gitnexus/test/unit/embedding-work-manifest.test.ts @@ -0,0 +1,261 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { EmbeddingByteBudget } from '../../src/core/analysis-resource-budget.js'; +import { + createEmbeddingWorkManifest, + fetchEmbeddingWorkUnit, + loadEmbeddingWorkManifest, + readEmbeddingWorkUnits, + type EmbeddingManifestSource, + type EmbeddingWorkKey, +} from '../../src/core/embeddings/embedding-work-manifest.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; + +const directories: string[] = []; +const temporary = async (): Promise => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-work-manifest-')); + directories.push(directory); + return directory; +}; +afterEach(async () => { + await Promise.all( + directories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +function synthetic( + count: number, + contentBytes = 1024, +): EmbeddingManifestSource & { scans: number; fetches: number; largestFetch: number } { + return { + scans: 0, + fetches: 0, + largestFetch: 0, + async *scan(label: string): AsyncGenerator { + this.scans++; + if (label !== 'Function') return; + for (let i = 0; i < count; i++) + yield { + id: `Function:f.ts:fn${String(i).padStart(8, '0')}`, + contentBytes, + metadataBytes: 1024, + }; + }, + async fetch(label: string, ids: readonly string[]): Promise { + this.fetches++; + this.largestFetch = Math.max(this.largestFetch, ids.length); + return ids + .map((id) => ({ + id, + name: id, + label, + filePath: 'f.ts', + content: 'x'.repeat(contentBytes), + startLine: 1, + endLine: 1, + })) + .reverse(); + }, + }; +} + +describe('durable embedding selection', () => { + it('resumes by cursor without rescanning or refetching completed nodes', async () => { + const directory = await temporary(); + const source = synthetic(130); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + expect(receipt.totalNodes).toBe(130); + expect(receipt.totalUnits).toBe(3); + const budget = new EmbeddingByteBudget(512 * 1024); + const units = readEmbeddingWorkUnits(directory, receipt); + const first = await units.next(); + if (first.done) throw new Error('Missing first unit'); + const fetched = await fetchEmbeddingWorkUnit(source, first.value.unit, budget); + expect(fetched.nodes[0].id).toBe(first.value.unit.nodes[0].id); + fetched.release(); + await units.return(undefined); + const scans = source.scans; + const reloaded = await loadEmbeddingWorkManifest(directory, 'graph-a'); + const remaining: number[] = []; + for await (const item of readEmbeddingWorkUnits(directory, reloaded, first.value.next)) { + remaining.push(item.unit.ordinal); + const loaded = await fetchEmbeddingWorkUnit(source, item.unit, budget); + expect(loaded.nodes.some((node) => node.id === first.value.unit.nodes[0].id)).toBe(false); + loaded.release(); + } + expect(remaining).toEqual([1, 2]); + expect(source.scans).toBe(scans); + expect(source.fetches).toBe(3); + expect(budget.usedBytes).toBe(0); + }); + + it('bounds fetched payloads independently of a lazily generated corpus size', async () => { + const peaks: number[] = []; + for (const count of [10_000, 100_000, 1_000_000]) { + const directory = await temporary(); + const source = synthetic(count); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + const budget = new EmbeddingByteBudget(512 * 1024); + let read = 0; + for await (const { unit } of readEmbeddingWorkUnits(directory, receipt)) { + const fetched = await fetchEmbeddingWorkUnit(source, unit, budget); + read += fetched.nodes.length; + fetched.release(); + } + expect(read).toBe(count); + expect(source.largestFetch).toBe(64); + expect(budget.usedBytes).toBe(0); + peaks.push(budget.peakBytes); + } + expect(peaks).toEqual([256 * 1024, 256 * 1024, 256 * 1024]); + }, 30_000); + + it('gives an oversized node its own stable unit and admits it explicitly', async () => { + const directory = await temporary(); + const source = synthetic(2, 2 * 1024 * 1024); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + expect(receipt.totalUnits).toBe(2); + for await (const { unit } of readEmbeddingWorkUnits(directory, receipt)) { + const before = source.fetches; + await expect( + fetchEmbeddingWorkUnit(source, unit, new EmbeddingByteBudget(1024)), + ).rejects.toThrow('requires'); + expect(source.fetches).toBe(before); + const fetched = await fetchEmbeddingWorkUnit( + source, + unit, + new EmbeddingByteBudget(5 * 1024 * 1024), + ); + expect(fetched.nodes[0].content.length).toBe(2 * 1024 * 1024); + fetched.release(); + } + }); + + it('rejects cross-label duplicate identities before publishing a selection', async () => { + const directory = await temporary(); + const source = synthetic(0); + source.scan = async function* (label) { + if (label === 'Function' || label === 'Class') + yield { id: 'shared-id', contentBytes: 1, metadataBytes: 1024 }; + }; + await expect(createEmbeddingWorkManifest(directory, 'graph-a', source)).rejects.toThrow( + 'duplicate graph node identity', + ); + expect(source.fetches).toBe(0); + await expect(fs.stat(path.join(directory, 'manifest.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('does not substitute File fallback when a symbol scan fails', async () => { + const directory = await temporary(); + const labels: string[] = []; + const source = synthetic(0); + source.scan = async function* (label) { + labels.push(label); + throw new Error('graph read failed'); + }; + await expect(createEmbeddingWorkManifest(directory, 'graph-a', source)).rejects.toThrow( + 'graph read failed', + ); + expect(labels).not.toContain('File'); + }); + + it('selects File only after every symbol label is empty', async () => { + const directory = await temporary(); + const source = synthetic(0); + source.scan = async function* (label) { + if (label === 'File') yield { id: 'File:README.md', contentBytes: 12, metadataBytes: 1024 }; + }; + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + for await (const result of readEmbeddingWorkUnits(directory, receipt)) { + expect(result.unit.label).toBe('File'); + break; + } + expect(receipt.totalNodes).toBe(1); + }); + + it('rejects source drift and releases its reserved capacity on a failed fetch', async () => { + const directory = await temporary(); + const source = synthetic(1); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + const iterator = readEmbeddingWorkUnits(directory, receipt); + const first = await iterator.next(); + if (first.done) throw new Error('Missing unit'); + await iterator.return(undefined); + const budget = new EmbeddingByteBudget(4096); + source.fetch = async () => []; + await expect(fetchEmbeddingWorkUnit(source, first.value.unit, budget)).rejects.toThrow( + 'missing or duplicate', + ); + expect(budget.usedBytes).toBe(0); + source.fetch = async () => [ + { + id: first.value.unit.nodes[0].id, + label: 'Function', + name: 'f', + filePath: 'f.ts', + content: 'changed', + }, + ]; + await expect(fetchEmbeddingWorkUnit(source, first.value.unit, budget)).rejects.toThrow( + 'graph changed', + ); + expect(budget.usedBytes).toBe(0); + }); + + it('preserves and rejects corrupt or foreign selections', async () => { + const directory = await temporary(); + await createEmbeddingWorkManifest(directory, 'graph-a', synthetic(1)); + await expect(loadEmbeddingWorkManifest(directory, 'graph-b')).rejects.toThrow('identity'); + await fs.appendFile(path.join(directory, 'manifest.jsonl'), '{}\n'); + await expect(loadEmbeddingWorkManifest(directory, 'graph-a')).rejects.toThrow('size mismatch'); + expect((await fs.stat(path.join(directory, 'manifest.jsonl'))).size).toBeGreaterThan(0); + }); + + it('rejects a misaligned durable cursor', async () => { + const directory = await temporary(); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', synthetic(65)); + await expect( + readEmbeddingWorkUnits(directory, receipt, { unit: 1, offset: 0 }).next(), + ).rejects.toThrow('Invalid embedding work unit'); + }); +}); + +describe('compact embedding receipts', () => { + it.skipIf(process.platform === 'win32')( + 'refuses symlink receipts and job directories', + async () => { + const directory = await temporary(); + const outside = await temporary(); + await writeEmbeddingJobRecord(outside, 'job.json', { cursor: 1 }); + await fs.symlink(path.join(outside, 'job.json'), path.join(directory, 'job.json')); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('regular file'); + await fs.symlink(outside, path.join(directory, 'linked-job')); + await expect( + readEmbeddingJobRecord(path.join(directory, 'linked-job'), 'job.json'), + ).rejects.toThrow('owned directory'); + }, + ); + + it('checks the checksum and bounds record sizes', async () => { + const directory = await temporary(); + await writeEmbeddingJobRecord(directory, 'job.json', { cursor: 3 }); + expect(await readEmbeddingJobRecord(directory, 'job.json')).toEqual({ cursor: 3 }); + const file = path.join(directory, 'job.json'); + await fs.writeFile(file, (await fs.readFile(file, 'utf8')).replace('"cursor":3', '"cursor":4')); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('checksum'); + await expect(writeEmbeddingJobRecord(directory, '../job.json', {})).rejects.toThrow('name'); + await expect( + writeEmbeddingJobRecord(directory, 'job.json', { padding: 'x'.repeat(40_000) }), + ).rejects.toThrow('size limit'); + await fs.writeFile(file, 'x'.repeat(40_000)); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('size limit'); + }); +}); From dd2f4906deed29b98411e1a21ff5201b7370db91 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 11:16:05 +0000 Subject: [PATCH 04/67] feat(embeddings): persist resumable vector jobs Keep graph readers independent of vector writes. Acknowledge units only after complete chunk verification and a native checkpoint. --- gitnexus/bench/analyze-resume/README.md | 17 + .../analyze-resume/vector-store-baseline.json | 60 ++ .../bench/analyze-resume/vector-store.mjs | 168 ++++++ .../core/embeddings/embedding-job-identity.ts | 70 +++ .../embeddings/embedding-work-manifest.ts | 14 +- gitnexus/src/core/lbug/lbug-config.ts | 21 +- gitnexus/src/core/lbug/vector-store.ts | 515 ++++++++++++++++++ gitnexus/src/storage/embedding-job-journal.ts | 339 ++++++++++++ gitnexus/src/storage/embedding-job.ts | 7 +- .../embedding-vector-store.test.ts | 186 +++++++ .../test/unit/embedding-job-journal.test.ts | 261 +++++++++ .../test/unit/lbug-explicit-budget.test.ts | 46 ++ gitnexus/vitest.config.ts | 5 + 13 files changed, 1700 insertions(+), 9 deletions(-) create mode 100644 gitnexus/bench/analyze-resume/vector-store-baseline.json create mode 100644 gitnexus/bench/analyze-resume/vector-store.mjs create mode 100644 gitnexus/src/core/embeddings/embedding-job-identity.ts create mode 100644 gitnexus/src/core/lbug/vector-store.ts create mode 100644 gitnexus/src/storage/embedding-job-journal.ts create mode 100644 gitnexus/test/integration/embedding-vector-store.test.ts create mode 100644 gitnexus/test/unit/embedding-job-journal.test.ts create mode 100644 gitnexus/test/unit/lbug-explicit-budget.test.ts diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 6070c9519..e60a9a6e2 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -23,3 +23,20 @@ These measurements establish native feasibility. They do not establish end-to-end analyze throughput, crash recovery, model throughput, or total RSS bounds. Run the application regression matrix before making those claims. Windows handle behavior and local-model execution also need their own runs. + +Measure the implemented vector store and journal after building: + +```sh +npm run build +node bench/analyze-resume/vector-store.mjs --check +``` + +This alternates five baseline and candidate runs over 512 nodes and 256-dimensional +vectors. The baseline uses the existing prepared DELETE/CREATE write sequence. +The candidate includes node receipts, complete chunk-set validation, a native +checkpoint per 64-node unit, and fsynced intent/acknowledgement journal updates. +Schema creation and final index construction are outside the timed region for +both paths. This isolates write overhead; it does not measure inference or graph +ingestion. `--check` rejects a candidate median above the baseline median. +The initial Linux/Node 22 run in `vector-store-baseline.json` measured a 0.470 +candidate/baseline median ratio, including the durability journal. diff --git a/gitnexus/bench/analyze-resume/vector-store-baseline.json b/gitnexus/bench/analyze-resume/vector-store-baseline.json new file mode 100644 index 000000000..092f73fed --- /dev/null +++ b/gitnexus/bench/analyze-resume/vector-store-baseline.json @@ -0,0 +1,60 @@ +{ + "node": "v22.23.3", + "platform": "linux", + "nodes": 512, + "dimensions": 256, + "unitNodes": 64, + "samples": [ + { + "trial": 0, + "mode": "baseline", + "ms": 3712.8183809999996 + }, + { + "trial": 0, + "mode": "durable", + "ms": 1684.6306610000001 + }, + { + "trial": 1, + "mode": "durable", + "ms": 1745.7803060000006 + }, + { + "trial": 1, + "mode": "baseline", + "ms": 3620.8808309999995 + }, + { + "trial": 2, + "mode": "baseline", + "ms": 3750.5283279999985 + }, + { + "trial": 2, + "mode": "durable", + "ms": 1692.780707 + }, + { + "trial": 3, + "mode": "durable", + "ms": 2035.2204090000014 + }, + { + "trial": 3, + "mode": "baseline", + "ms": 3115.5813979999984 + }, + { + "trial": 4, + "mode": "baseline", + "ms": 3907.7676600000013 + }, + { + "trial": 4, + "mode": "durable", + "ms": 1761.2801880000006 + } + ], + "ratio": 0.4702035291933125 +} diff --git a/gitnexus/bench/analyze-resume/vector-store.mjs b/gitnexus/bench/analyze-resume/vector-store.mjs new file mode 100644 index 000000000..4fa174de1 --- /dev/null +++ b/gitnexus/bench/analyze-resume/vector-store.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env node +// Actual vector-store/journal write cost, excluding model and graph ingestion. +// npm run build && node bench/analyze-resume/vector-store.mjs --check +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { VectorStore } from '../../dist/core/lbug/vector-store.js'; +import { EmbeddingJobJournal } from '../../dist/storage/embedding-job-journal.js'; +import { openLbugConnection, closeLbugConnection } from '../../dist/core/lbug/lbug-config.js'; + +process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; +const nodes = 512; +const dimensions = 256; +const unitNodes = 64; +const identity = 'a'.repeat(64); +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-measure-')); +const samples = []; +const manifest = { + version: 1, + sourceIdentity: identity, + digest: 'b'.repeat(64), + bytes: nodes / unitNodes, + totalNodes: nodes, + totalUnits: nodes / unitNodes, + maximumUnitBytes: 1024, +}; +const chunk = (node) => ({ + nodeId: `Function:source.ts:node${node}`, + contentHash: 'same-input-hash', + chunkIndex: 0, + startLine: 1, + endLine: 2, + embedding: Array(dimensions).fill(0.125), +}); + +async function baseline(directory) { + const handle = await openLbugConnection(lbug, path.join(directory, 'vectors'), { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const prepared = new Map(); + async function query(sql, params) { + let result; + if (params) { + let statement = prepared.get(sql); + if (!statement) { + statement = await handle.conn.prepare(sql); + prepared.set(sql, statement); + } + result = await handle.conn.execute(statement, params); + } else result = await handle.conn.query(sql); + const parts = Array.isArray(result) ? result : [result]; + const rows = []; + try { + for (const part of parts) while (part.hasNext()) rows.push(part.getNextSync()); + } finally { + for (const part of parts) part.close(); + } + return rows; + } + try { + await query( + `CREATE NODE TABLE CodeEmbedding(id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${dimensions}], contentHash STRING, PRIMARY KEY(id))`, + ); + const start = performance.now(); + // The existing batchInsertEmbeddings path: prepared DELETE then CREATE, + // with each write auto-committed and a checkpoint after the full pipeline. + for (let offset = 0; offset < nodes; offset += unitNodes) { + const rows = Array.from({ length: unitNodes }, (_, i) => chunk(offset + i)); + for (const row of rows) + await query('MATCH (e:CodeEmbedding {id:$id}) DELETE e', { id: `${row.nodeId}:0` }); + for (const row of rows) + await query( + 'CREATE (:CodeEmbedding {id:$id, nodeId:$nodeId, chunkIndex:$chunkIndex, startLine:$startLine, endLine:$endLine, embedding:$embedding, contentHash:$contentHash})', + { id: `${row.nodeId}:0`, ...row }, + ); + } + await query('CHECKPOINT'); + const ms = performance.now() - start; + assert.equal(Number((await query('MATCH (e:CodeEmbedding) RETURN count(e) AS n'))[0].n), nodes); + return ms; + } finally { + await closeLbugConnection(handle); + } +} + +async function durable(directory) { + const store = await VectorStore.open({ + directory, + identity, + dimensions, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + mode: 'create', + }); + try { + const start = performance.now(); + const journal = await EmbeddingJobJournal.create(directory, identity, manifest); + for (let offset = 0; offset < nodes; offset += unitNodes) { + const ordinal = offset / unitNodes; + const rows = Array.from({ length: unitNodes }, (_, i) => chunk(offset + i)); + await journal.begin({ + start: { unit: ordinal, offset: ordinal }, + next: { unit: ordinal + 1, offset: ordinal + 1 }, + nodes: unitNodes, + origin: 'manifest', + }); + await store.beginNodes(rows); + await store.appendBatch(rows); + const receipts = await store.finishNodes( + rows.map(({ nodeId, contentHash }) => ({ nodeId, contentHash, chunks: 1 })), + ); + await store.checkpointUnit(receipts); + await journal.acknowledge({ reusedNodes: 0, chunks: unitNodes }); + } + const ms = performance.now() - start; + const state = journal.snapshot(); + assert.equal(state.completedNodes, nodes); + assert.equal(state.chunks, nodes); + assert.equal(state.active, null); + assert.equal(await store.seal(nodes, nodes), 'exact'); + return ms; + } finally { + await store.close(); + } +} + +try { + for (let trial = 0; trial < 5; trial++) { + for (const mode of trial % 2 ? ['durable', 'baseline'] : ['baseline', 'durable']) { + const directory = path.join(root, `${trial}-${mode}`); + await fs.mkdir(directory); + samples.push({ + trial, + mode, + ms: await (mode === 'baseline' ? baseline : durable)(directory), + }); + } + } + const median = (mode) => + samples + .filter((s) => s.mode === mode) + .map((s) => s.ms) + .sort((a, b) => a - b)[2]; + const ratio = median('durable') / median('baseline'); + process.stdout.write( + JSON.stringify( + { + node: process.version, + platform: process.platform, + nodes, + dimensions, + unitNodes, + samples, + ratio, + }, + null, + 2, + ) + '\n', + ); + if (process.argv.includes('--check')) + assert.ok(ratio <= 1, `Durable native writes regressed: ${ratio.toFixed(3)}x`); +} finally { + await fs.rm(root, { recursive: true, force: true }); +} diff --git a/gitnexus/src/core/embeddings/embedding-job-identity.ts b/gitnexus/src/core/embeddings/embedding-job-identity.ts new file mode 100644 index 000000000..49f8cf905 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job-identity.ts @@ -0,0 +1,70 @@ +import { createHash } from 'node:crypto'; +import type { EmbeddingIdentity } from './embedding-identity.js'; +import type { EmbeddingConfig } from './types.js'; + +export interface EmbeddingGraphIdentity { + generation: string; + coveredContent: string; + revision: string; + analyzer: string; + parser: string; + schema: string; +} + +const digest = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); + +/** Explicit fields exclude credentials and scheduling settings from persisted identity. */ +export function embeddingJobIdentity( + graph: EmbeddingGraphIdentity, + vector: EmbeddingIdentity, + textVersion: string, + config: Pick< + EmbeddingConfig, + 'chunkSize' | 'overlap' | 'maxSnippetLength' | 'maxDescriptionLength' | 'maxInputChars' + >, +): { identity: string; sourceIdentity: string; vectorIdentity: string; inputIdentity: string } { + const source = [ + graph.generation, + graph.coveredContent, + graph.revision, + graph.analyzer, + graph.parser, + graph.schema, + ]; + if ( + source.some((value) => typeof value !== 'string' || !value) || + !vector.model || + !/^(local|http:[a-f0-9]{64})$/.test(vector.provider) || + !Number.isSafeInteger(vector.dimensions) || + vector.dimensions < 1 || + !textVersion || + ![ + config.chunkSize, + config.overlap, + config.maxSnippetLength, + config.maxDescriptionLength, + config.maxInputChars, + ].every((n) => Number.isSafeInteger(n) && n >= 0) || + config.chunkSize < 1 || + config.overlap >= config.chunkSize || + config.maxInputChars < 2 + ) + throw new Error('Incomplete embedding job identity'); + const sourceIdentity = digest(source); + const vectorIdentity = digest([vector.provider, vector.model, vector.dimensions]); + const inputIdentity = digest([ + textVersion, + config.chunkSize, + config.overlap, + config.maxSnippetLength, + config.maxDescriptionLength, + config.maxInputChars, + ]); + return { + sourceIdentity, + vectorIdentity, + inputIdentity, + identity: digest([1, sourceIdentity, vectorIdentity, inputIdentity]), + }; +} diff --git a/gitnexus/src/core/embeddings/embedding-work-manifest.ts b/gitnexus/src/core/embeddings/embedding-work-manifest.ts index 2de60655e..1c14018e7 100644 --- a/gitnexus/src/core/embeddings/embedding-work-manifest.ts +++ b/gitnexus/src/core/embeddings/embedding-work-manifest.ts @@ -160,15 +160,19 @@ async function verifyUniqueKeys(files: string[]): Promise { let previous: string | undefined; for (;;) { let selected = -1; + let selectedHead: { id: string; bytes: Buffer } | undefined; for (let i = 0; i < heads.length; i++) { + const candidate = heads[i]; if ( - heads[i] && - (selected < 0 || Buffer.compare(heads[i]!.bytes, heads[selected]!.bytes) < 0) - ) + candidate && + (!selectedHead || Buffer.compare(candidate.bytes, selectedHead.bytes) < 0) + ) { selected = i; + selectedHead = candidate; + } } - if (selected < 0) break; - const id = heads[selected]!.id; + if (!selectedHead) break; + const id = selectedHead.id; if (id === previous) throw new Error( 'Embedding read: duplicate graph node identity; rebuild with analyze --force', diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index 2175da5e1..f4fb73e72 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -847,6 +847,9 @@ type LbugModule = typeof lbug; export interface LbugDatabaseOptions { readOnly?: boolean; throwOnWalReplayFailure?: boolean; + /** Explicit per-handle reservation; omitted preserves the existing policy. */ + bufferPoolBytes?: number; + threads?: number; } export interface LbugConnectionHandle { @@ -946,10 +949,15 @@ export function createLbugDatabase( databasePath: string, options: LbugDatabaseOptions = {}, ): lbug.Database { + if ( + options.bufferPoolBytes !== undefined && + (!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes <= 0) + ) + throw new Error('Explicit LadybugDB buffer pool must be a positive byte count'); // .d.ts declares fewer args than the native constructor accepts. return new (lbugModule.Database as any)( databasePath, - resolveBufferManagerSize(), // bufferManagerSize (#2557: default min(2 GiB, 80% RAM); GITNEXUS_LBUG_BUFFER_POOL_SIZE overrides; 0 restores the native 80%-of-RAM default) + options.bufferPoolBytes ?? resolveBufferManagerSize(), // Existing callers retain the environment/default policy. false, // enableCompression (pinned for v0.16.0) options.readOnly ?? false, LBUG_MAX_DB_SIZE, @@ -1145,11 +1153,20 @@ export async function openLbugConnection( databasePath: string, options: LbugDatabaseOptions = {}, ): Promise { + if ( + options.threads !== undefined && + (!Number.isSafeInteger(options.threads) || options.threads < 1) + ) + throw new Error('LadybugDB threads must be a positive integer'); const safePath = toNativeSafePath(databasePath); let db: lbug.Database | undefined; try { db = await openWithLockRetry(() => createLbugDatabase(lbugModule, safePath, options), safePath); - return { db, conn: new lbugModule.Connection(db) }; + const conn = + options.threads === undefined + ? new lbugModule.Connection(db) + : new lbugModule.Connection(db, options.threads); + return { db, conn }; } catch (err) { if (db) await db.close().catch(() => {}); throw err; diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts new file mode 100644 index 000000000..ce0e8a43e --- /dev/null +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -0,0 +1,515 @@ +/** An exclusively owned vector artifact; never opens the graph singleton. */ +import lbug, { type LbugValue } from '@ladybugdb/core'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { openLbugConnection, type LbugConnectionHandle } from './lbug-config.js'; +import { ExtensionManager } from './extension-loader.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import { embeddingJobDirectory } from '../../storage/embedding-job.js'; + +export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v1'; +const BLOCK_ROWS = 256; +const MAX_CHUNKS = 2147483647; + +export interface StoredVectorChunk { + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; +} +export interface VectorNodeReceipt { + nodeId: string; + contentHash: string; + chunks: number; + complete: boolean; +} +export interface VectorNodeInput { + nodeId: string; + contentHash: string; +} +export type VectorWriteChunk = StoredVectorChunk & VectorNodeInput; +export interface VectorStoreOptions { + directory: string; + identity: string; + dimensions: number; + bufferPoolBytes: number; + threads: number; + mode: 'create' | 'resume' | 'read'; + /** Maximum JS vector batch retained by the caller and passed into native. */ + batchBytes: number; +} + +type Row = Record; +const nonnegative = (n: unknown): n is number => + typeof n === 'number' && Number.isSafeInteger(n) && n >= 0; +const count = (value: unknown): number => { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid vector-store count'); + const n = Number(value); + if (!nonnegative(n)) throw new Error('Invalid vector-store count'); + return n; +}; + +export class VectorStore { + private readonly prepared = new Map(); + private busy = false; + private closed = false; + private poisoned = false; + private sealed = false; + + private constructor( + private readonly handle: LbugConnectionHandle, + readonly options: Readonly, + ) {} + + static async open(options: VectorStoreOptions): Promise { + if ( + !/^[a-f0-9]{64}$/.test(options.identity) || + !Number.isSafeInteger(options.dimensions) || + options.dimensions < 1 || + !Number.isSafeInteger(options.batchBytes) || + options.batchBytes < options.dimensions * 16 || + !Number.isSafeInteger(options.bufferPoolBytes) || + options.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(options.threads) || + options.threads < 1 || + !['create', 'resume', 'read'].includes(options.mode) + ) + throw new Error('Invalid vector-store resource reservation or identity'); + const directory = await embeddingJobDirectory(options.directory); + const file = path.join(directory, 'vectors'); + for (const suffix of ['', '.wal', '.lock']) { + try { + const stat = await fs.lstat(file + suffix); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector artifact must be an owned regular file'); + if (suffix === '' && options.mode === 'create') + throw new Error('Vector artifact already exists'); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' || + (suffix === '' && options.mode !== 'create') + ) + throw error; + } + } + const handle = await openLbugConnection(lbug, file, { + readOnly: options.mode === 'read', + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }); + const store = new VectorStore(handle, Object.freeze({ ...options, directory })); + try { + if (options.mode === 'create') { + await store.query( + `CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${options.dimensions}], contentHash STRING, PRIMARY KEY(id))`, + ); + await store.query( + 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, PRIMARY KEY(nodeId))', + ); + await store.query( + 'CREATE NODE TABLE EmbeddingArtifact (id STRING, identity STRING, schema STRING, dimensions INT64, complete BOOL, indexKind STRING, PRIMARY KEY(id))', + ); + await store.query( + "CREATE (:EmbeddingArtifact {id:'artifact', identity:$identity, schema:$schema, dimensions:$dimensions, complete:false, indexKind:'pending'})", + { + identity: options.identity, + schema: VECTOR_STORE_SCHEMA, + dimensions: options.dimensions, + }, + ); + await store.query('CHECKPOINT'); + } else { + const rows = await store.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.identity AS identity, a.schema AS schema, a.dimensions AS dimensions, a.complete AS complete", + ); + if ( + rows.length !== 1 || + rows[0].identity !== options.identity || + rows[0].schema !== VECTOR_STORE_SCHEMA || + count(rows[0].dimensions) !== options.dimensions + ) + throw new Error('Vector artifact identity mismatch; artifacts retained'); + store.sealed = rows[0].complete === true; + if (options.mode === 'read' && !store.sealed) + throw new Error('Vector artifact is not complete'); + } + return store; + } catch (error) { + await store.close().catch(() => {}); + throw error; + } + } + + private async exclusive(action: () => Promise, write = false): Promise { + if (this.closed || this.poisoned) throw new Error('Vector store must be reopened'); + if (this.busy) throw new Error('Concurrent vector-store access is not allowed'); + if (write && (this.options.mode === 'read' || this.sealed)) + throw new Error('Published vector artifact is immutable'); + this.busy = true; + try { + return await action(); + } finally { + this.busy = false; + } + } + + private async query(sql: string, parameters?: Record): Promise { + let result: lbug.QueryResult | lbug.QueryResult[]; + if (parameters) { + let statement = this.prepared.get(sql); + if (!statement) { + statement = await this.handle.conn.prepare(sql); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + this.prepared.set(sql, statement); + } + result = await this.handle.conn.execute(statement, parameters); + } else result = await this.handle.conn.query(sql); + const results = Array.isArray(result) ? result : [result]; + const rows: Row[] = []; + try { + for (const part of results) + while (part.hasNext()) { + if (rows.length >= BLOCK_ROWS) + throw new Error('Vector query exceeded its bounded result size'); + rows.push(part.getNextSync()); + } + return rows; + } finally { + for (const part of results) part.close(); + } + } + + private async transaction(action: () => Promise): Promise { + await this.query('BEGIN TRANSACTION'); + try { + await action(); + await this.query('COMMIT'); + } catch (error) { + await this.query('ROLLBACK').catch(() => { + this.poisoned = true; + }); + throw error; + } + } + + private async receipt(nodeId: string): Promise { + return (await this.receipts([nodeId])).get(nodeId) ?? null; + } + + private async receipts(ids: readonly string[]): Promise> { + const wanted = new Set(ids); + const rows = await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) RETURN n.nodeId AS nodeId, n.contentHash AS contentHash, n.chunks AS chunks, n.complete AS complete', + { ids: [...ids] }, + ); + const receipts = new Map(); + for (const row of rows) { + const chunks = count(row.chunks); + if ( + typeof row.nodeId !== 'string' || + !wanted.has(row.nodeId) || + receipts.has(row.nodeId) || + typeof row.contentHash !== 'string' || + !row.contentHash || + chunks > MAX_CHUNKS || + typeof row.complete !== 'boolean' + ) + throw new Error('Corrupt vector node receipt'); + receipts.set(row.nodeId, { + nodeId: row.nodeId, + contentHash: row.contentHash, + chunks, + complete: row.complete, + }); + } + return receipts; + } + + /** Replay replaces the entire old group, using PKs rather than repeated table scans. */ + async beginNode(nodeId: string, contentHash: string): Promise { + return this.beginNodes([{ nodeId, contentHash }]); + } + + async beginNodes(nodes: readonly VectorNodeInput[]): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector node unit'); + for (const node of nodes) { + assertValidGraphNodeId(node.nodeId, 'Vector store'); + if (!node.contentHash) throw new Error('Missing embedding input hash'); + } + await this.exclusive(async () => { + const previous = await this.receipts(nodes.map((n) => n.nodeId)); + if (previous.size) { + // Clear the proof before deleting any row. The old high-water mark is + // retained until every old chunk has been removed, including on crash. + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=false', + { ids: [...previous.keys()] }, + ); + let ids: string[] = []; + const remove = async () => { + if (!ids.length) return; + await this.transaction(async () => { + await this.query('UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) DELETE e', { ids }); + }); + ids = []; + }; + for (const receipt of previous.values()) + for (let i = 0; i < receipt.chunks; i++) { + ids.push(`${receipt.nodeId}:${i}`); + if (ids.length === BLOCK_ROWS) await remove(); + } + await remove(); + } + await this.transaction(async () => { + for (const { nodeId, contentHash } of nodes) { + if (previous.has(nodeId)) + await this.query( + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.chunks=0', + { nodeId, contentHash }, + ); + else + await this.query( + 'CREATE (:EmbeddingNodeReceipt {nodeId:$nodeId, contentHash:$contentHash, chunks:0, complete:false})', + { nodeId, contentHash }, + ); + } + }); + }, true); + } + + async append( + nodeId: string, + contentHash: string, + chunks: readonly StoredVectorChunk[], + ): Promise { + return this.appendBatch(chunks.map((chunk) => ({ ...chunk, nodeId, contentHash }))); + } + + async appendBatch(chunks: readonly VectorWriteChunk[]): Promise { + if ( + !chunks.length || + chunks.length > BLOCK_ROWS || + chunks.reduce( + (bytes, chunk) => + bytes + + this.options.dimensions * 16 + + 128 + + 2 * (chunk.nodeId.length + chunk.contentHash.length), + 0, + ) > this.options.batchBytes + ) + throw new Error('Vector write exceeds its byte/row reservation'); + for (const chunk of chunks) { + assertValidGraphNodeId(chunk.nodeId, 'Vector store'); + if ( + !nonnegative(chunk.chunkIndex) || + chunk.chunkIndex >= MAX_CHUNKS || + !nonnegative(chunk.startLine) || + !nonnegative(chunk.endLine) || + chunk.endLine < chunk.startLine || + chunk.embedding.length !== this.options.dimensions || + chunk.embedding.some((n) => !Number.isFinite(n) || Math.abs(n) > 3.4028234663852886e38) + ) + throw new Error('Invalid embedding chunk identity, lines, or vector'); + } + await this.exclusive(async () => { + const ids = [...new Set(chunks.map((chunk) => chunk.nodeId))]; + if (ids.length > 64) throw new Error('Vector write crosses its node unit'); + const receipts = await this.receipts(ids); + for (const chunk of chunks) { + const receipt = receipts.get(chunk.nodeId); + if ( + !receipt || + receipt.complete || + receipt.contentHash !== chunk.contentHash || + chunk.chunkIndex !== receipt.chunks + ) + throw new Error('Vector chunks do not extend the active node contiguously'); + receipt.chunks++; + } + await this.transaction(async () => { + for (const chunk of chunks) + await this.query( + 'CREATE (:CodeEmbedding {id:$id, nodeId:$nodeId, chunkIndex:$chunkIndex, startLine:$startLine, endLine:$endLine, embedding:$embedding, contentHash:$contentHash})', + { id: `${chunk.nodeId}:${chunk.chunkIndex}`, ...chunk }, + ); + for (const receipt of receipts.values()) + await this.query('MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.chunks=$chunks', { + nodeId: receipt.nodeId, + chunks: receipt.chunks, + }); + }); + }, true); + } + + private async verify(receipt: VectorNodeReceipt): Promise { + for (let start = 0; start < receipt.chunks; start += BLOCK_ROWS) { + const ids = Array.from( + { length: Math.min(BLOCK_ROWS, receipt.chunks - start) }, + (_, i) => `${receipt.nodeId}:${start + i}`, + ); + const rows = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash', + { ids }, + ); + const seen = new Set(); + for (const row of rows) { + const i = count(row.chunkIndex); + if ( + row.nodeId !== receipt.nodeId || + row.contentHash !== receipt.contentHash || + i < start || + i >= start + ids.length || + row.id !== `${receipt.nodeId}:${i}` || + seen.has(String(row.id)) + ) + throw new Error('Vector node has an invalid chunk set'); + seen.add(String(row.id)); + } + if (seen.size !== ids.length) throw new Error('Vector node is missing chunks'); + } + } + + async finishNode( + nodeId: string, + contentHash: string, + chunks: number, + ): Promise { + return (await this.finishNodes([{ nodeId, contentHash, chunks }]))[0]; + } + + async finishNodes( + nodes: readonly (VectorNodeInput & { chunks: number })[], + ): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector completion unit'); + return this.exclusive(async () => { + const receipts = await this.receipts(nodes.map((n) => n.nodeId)); + for (const node of nodes) { + const receipt = receipts.get(node.nodeId); + if ( + !receipt || + receipt.contentHash !== node.contentHash || + receipt.chunks !== node.chunks || + node.chunks < 1 + ) + throw new Error('Vector node coverage does not match its generated chunks'); + await this.verify(receipt); + } + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', + { ids: nodes.map((n) => n.nodeId) }, + ); + return nodes.map((node) => ({ ...node, complete: true })); + }, true); + } + + async reusable(nodeId: string, contentHash: string): Promise { + return this.exclusive(async () => { + const receipt = await this.receipt(nodeId); + if (!receipt?.complete || receipt.contentHash !== contentHash || receipt.chunks < 1) + return null; + await this.verify(receipt); + return receipt; + }); + } + + /** Durability must precede the filesystem journal's acknowledgement. */ + async checkpointUnit(nodes: readonly VectorNodeReceipt[]): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector durability unit'); + await this.exclusive(async () => { + const receipts = await this.receipts(nodes.map((n) => n.nodeId)); + for (const node of nodes) { + const stored = receipts.get(node.nodeId); + if ( + !stored?.complete || + stored.contentHash !== node.contentHash || + stored.chunks !== node.chunks || + stored.chunks < 1 + ) + throw new Error('Cannot acknowledge incomplete vector unit'); + } + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, true); + } + + /** Runs only once all manifest and failed units have complete coverage. */ + async seal(totalNodes: number, totalChunks: number): Promise<'hnsw' | 'exact'> { + if (!nonnegative(totalNodes) || !nonnegative(totalChunks)) + throw new Error('Invalid vector coverage totals'); + return this.exclusive(async () => { + const nodes = await this.query( + 'MATCH (n:EmbeddingNodeReceipt) RETURN count(n) AS nodes, sum(n.chunks) AS chunks, sum(CASE WHEN n.complete THEN 0 ELSE 1 END) AS pending', + ); + const vectors = await this.query('MATCH (e:CodeEmbedding) RETURN count(e) AS chunks'); + if ( + count(nodes[0].nodes) !== totalNodes || + count(nodes[0].chunks ?? 0) !== totalChunks || + count(nodes[0].pending ?? 0) !== 0 || + count(vectors[0].chunks) !== totalChunks + ) + throw new Error('Vector artifact coverage is incomplete'); + const manager = new ExtensionManager(); + let kind: 'hnsw' | 'exact' = 'exact'; + if ( + totalChunks && + (await manager.ensure((sql) => this.query(sql), 'vector', 'vector search')) + ) { + try { + await this.query( + "CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')", + ); + kind = 'hnsw'; + } catch (error) { + // A complete artifact may be searched exactly if optional ANN fails. + // Native corruption/durability errors still surface at checkpoint. + if (/already exists/i.test(String(error))) kind = 'hnsw'; + } + } + await this.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) SET a.complete=true, a.indexKind=$kind", + { kind }, + ); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + this.sealed = true; + return kind; + }, true); + } + + async close(): Promise { + if (this.closed) return; + if (this.busy) throw new Error('Cannot close an active vector operation'); + this.closed = true; + this.prepared.clear(); + try { + await this.handle.conn.close(); + } finally { + await this.handle.db.close(); + } + } +} diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts new file mode 100644 index 000000000..5538b5c29 --- /dev/null +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -0,0 +1,339 @@ +/** Compact scheduling journal. The caller owns the checkout's exclusive job lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type { + EmbeddingManifestCursor, + EmbeddingManifestReceipt, +} from '../core/embeddings/embedding-work-manifest.js'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from './embedding-job.js'; + +export interface EmbeddingUnitIntent { + start: EmbeddingManifestCursor; + next: EmbeddingManifestCursor; + nodes: number; + origin: 'manifest' | 'failed'; +} + +export interface EmbeddingJobState { + version: 1; + identity: string; + manifestDigest: string; + totalUnits: number; + totalNodes: number; + manifestBytes: number; + cursor: EmbeddingManifestCursor; + completedUnits: number; + completedNodes: number; + reusedUnits: number; + reusedNodes: number; + failedUnits: number; + failedNodes: number; + chunks: number; + active: EmbeddingUnitIntent | null; + phase: 'embedding' | 'indexing' | 'ready'; +} + +export type EmbeddingFailureCause = 'provider' | 'input' | 'cancelled'; +interface FailedUnit { + version: 1; + identity: string; + manifestDigest: string; + intent: EmbeddingUnitIntent; + cause: EmbeddingFailureCause; + attempt: string; +} + +const DIGEST = /^[a-f0-9]{64}$/; +const FAILED = /^failed-(\d{16})\.json$/; +const integer = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +const filename = (unit: number): string => `failed-${String(unit).padStart(16, '0')}.json`; +const clone = (value: T): T => structuredClone(value); + +function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): void { + if ( + !intent || + !intent.start || + !intent.next || + !integer(intent.start.unit) || + intent.start.unit >= state.totalUnits || + !integer(intent.start.offset) || + !integer(intent.next.offset) || + intent.next.offset <= intent.start.offset || + intent.next.offset > state.manifestBytes || + intent.next.unit !== intent.start.unit + 1 || + !integer(intent.nodes) || + intent.nodes < 1 || + intent.nodes > 64 || + !['manifest', 'failed'].includes(intent.origin) + ) + throw new Error('Invalid embedding unit intent'); +} + +function validateState( + value: unknown, + identity: string, + manifest: EmbeddingManifestReceipt, +): asserts value is EmbeddingJobState { + const s = value as EmbeddingJobState; + if ( + !s || + s.version !== 1 || + s.identity !== identity || + s.manifestDigest !== manifest.digest || + s.totalNodes !== manifest.totalNodes || + s.totalUnits !== manifest.totalUnits || + s.manifestBytes !== manifest.bytes || + !s.cursor || + !integer(s.cursor.unit) || + s.cursor.unit > s.totalUnits || + !integer(s.cursor.offset) || + s.cursor.offset > manifest.bytes || + (s.cursor.unit === 0 && s.cursor.offset !== 0) || + (s.cursor.unit === s.totalUnits && s.cursor.offset !== manifest.bytes) || + ![ + s.completedUnits, + s.completedNodes, + s.reusedUnits, + s.reusedNodes, + s.failedUnits, + s.failedNodes, + s.chunks, + ].every(integer) || + s.completedUnits + s.reusedUnits + s.failedUnits > s.totalUnits || + s.completedNodes + s.reusedNodes + s.failedNodes > s.totalNodes || + !['embedding', 'indexing', 'ready'].includes(s.phase) || + s.active === undefined + ) + throw new Error('Embedding job identity or coverage is invalid; artifacts retained'); + if (s.active) { + validateIntent(s.active, s); + const active = s.active; + if ( + active.origin === 'manifest' + ? active.start.unit !== s.cursor.unit || active.start.offset !== s.cursor.offset + : active.next.unit > s.cursor.unit + ) + throw new Error('Embedding job active cursor is inconsistent'); + } + const settled = s.completedUnits + s.reusedUnits + s.failedUnits; + if ( + settled + (s.active?.origin === 'failed' ? 1 : 0) !== s.cursor.unit || + (s.phase !== 'embedding' && + (s.active !== null || + s.failedUnits !== 0 || + settled !== s.totalUnits || + s.completedNodes + s.reusedNodes !== s.totalNodes)) + ) + throw new Error('Embedding job accounting is inconsistent'); +} + +/** No model, DB, source text, or growing pending-ID list is retained here. */ +export class EmbeddingJobJournal { + private busy = false; + private uncertain = false; + /** Failed units are attempted at most once per invocation, even if readdir sees a replacement. */ + readonly attempt = randomUUID(); + + private constructor( + readonly directory: string, + private state: EmbeddingJobState, + private readonly manifest: EmbeddingManifestReceipt, + ) {} + + static async create( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + if (!DIGEST.test(identity) || !DIGEST.test(manifest.digest)) + throw new Error('Invalid embedding job identity'); + const canonical = await embeddingJobDirectory(directory); + try { + await fs.lstat(path.join(canonical, 'job.json')); + throw new Error('Embedding job already exists'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const state: EmbeddingJobState = { + version: 1, + identity, + manifestDigest: manifest.digest, + totalUnits: manifest.totalUnits, + totalNodes: manifest.totalNodes, + manifestBytes: manifest.bytes, + cursor: { unit: 0, offset: 0 }, + completedUnits: 0, + completedNodes: 0, + reusedUnits: 0, + reusedNodes: 0, + failedUnits: 0, + failedNodes: 0, + chunks: 0, + active: null, + phase: 'embedding', + }; + validateState(state, identity, manifest); + await writeEmbeddingJobRecord(canonical, 'job.json', state); + return new EmbeddingJobJournal(canonical, state, clone(manifest)); + } + + static async open( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + if (!DIGEST.test(identity)) throw new Error('Invalid embedding job identity'); + const canonical = await embeddingJobDirectory(directory); + const state = await readEmbeddingJobRecord(canonical, 'job.json'); + validateState(state, identity, manifest); + return new EmbeddingJobJournal(canonical, state, clone(manifest)); + } + + snapshot(): EmbeddingJobState { + return clone(this.state); + } + + private async mutate(action: (next: EmbeddingJobState) => Promise): Promise { + if (this.busy) throw new Error('Concurrent embedding journal mutation'); + if (this.uncertain) + throw new Error('Embedding journal durability uncertain; reopen before retry'); + this.busy = true; + try { + const next = clone(this.state); + await action(next); + validateState(next, next.identity, this.manifest); + await writeEmbeddingJobRecord(this.directory, 'job.json', next); + this.state = next; + } catch (error) { + // Even a failed fsync can follow a successful rename. Never keep writing + // using an in-memory cursor that may differ from the durable one. + this.uncertain = true; + throw error; + } finally { + this.busy = false; + } + } + + async begin(intent: EmbeddingUnitIntent): Promise { + const requested = clone(intent); + await this.mutate(async (next) => { + if (next.active || next.phase !== 'embedding') + throw new Error('Embedding job already active or complete'); + validateIntent(requested, next); + if (requested.origin === 'failed') { + const record = await this.readFailed(filename(requested.start.unit)); + if ( + record.intent.start.unit !== requested.start.unit || + record.intent.start.offset !== requested.start.offset || + record.intent.next.unit !== requested.next.unit || + record.intent.next.offset !== requested.next.offset || + record.intent.nodes !== requested.nodes + ) + throw new Error('Failed embedding unit cursor mismatch'); + next.failedUnits--; + next.failedNodes -= requested.nodes; + } + next.active = requested; + }); + } + + /** Call only after the vector store has verified and checkpointed this unit. */ + async acknowledge(result: { reusedNodes: number; chunks: number }): Promise { + await this.mutate(async (next) => { + const active = next.active; + if ( + !active || + !integer(result.reusedNodes) || + result.reusedNodes > active.nodes || + !integer(result.chunks) + ) + throw new Error('Invalid embedding unit acknowledgement'); + // If the process dies after unlink, the still-active intent owns replay. + await fs.rm(path.join(this.directory, filename(active.start.unit)), { force: true }); + await syncEmbeddingJobDirectory(this.directory); + if (result.reusedNodes === active.nodes) next.reusedUnits++; + else next.completedUnits++; + next.reusedNodes += result.reusedNodes; + next.completedNodes += active.nodes - result.reusedNodes; + next.chunks += result.chunks; + if (active.origin === 'manifest') next.cursor = active.next; + next.active = null; + }); + } + + /** Input/provider failures stay durable; storage/durability failures must abort. */ + async fail(cause: EmbeddingFailureCause): Promise { + await this.mutate(async (next) => { + const active = next.active; + if (!active || !['provider', 'input', 'cancelled'].includes(cause)) + throw new Error('Invalid embedding failure'); + const record: FailedUnit = { + version: 1, + identity: next.identity, + manifestDigest: next.manifestDigest, + intent: { ...active, origin: 'failed' }, + cause, + attempt: this.attempt, + }; + await writeEmbeddingJobRecord(this.directory, filename(active.start.unit), record); + next.failedUnits++; + next.failedNodes += active.nodes; + if (active.origin === 'manifest') next.cursor = active.next; + next.active = null; + }); + } + + private async readFailed(name: string): Promise { + const record = (await readEmbeddingJobRecord(this.directory, name)) as FailedUnit; + if ( + !record || + record.version !== 1 || + record.identity !== this.state.identity || + record.manifestDigest !== this.state.manifestDigest || + !['provider', 'input', 'cancelled'].includes(record.cause) || + typeof record.attempt !== 'string' + ) + throw new Error('Invalid failed embedding unit'); + validateIntent(record.intent, this.state); + if (record.intent.origin !== 'failed' || name !== filename(record.intent.start.unit)) + throw new Error('Failed embedding unit identity mismatch'); + return record; + } + + async *failed(): AsyncGenerator { + const directory = await fs.opendir(this.directory); + for await (const entry of directory) { + if (!FAILED.test(entry.name)) continue; + const record = await this.readFailed(entry.name); + if ( + record.attempt === this.attempt || + record.intent.start.unit === this.state.active?.start.unit + ) + continue; + if (record.intent.next.unit > this.state.cursor.unit) + throw new Error('Orphan failed unit has no active intent'); + yield clone(record.intent); + } + } + + async indexing(): Promise { + await this.mutate(async (next) => { + next.phase = 'indexing'; + }); + } + + /** The completed vector artifact must be closed and published before this call. */ + async ready(): Promise { + await this.mutate(async (next) => { + if (next.phase !== 'indexing') throw new Error('Embedding index has not been finalized'); + next.phase = 'ready'; + }); + } +} diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 4f6ff3d2a..15b2c3f28 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -6,6 +6,7 @@ import path from 'node:path'; const MAX_RECORD_BYTES = 32 * 1024; const RECORD_NAMES = new Set(['manifest.json', 'job.json']); +const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; export async function embeddingJobDirectory(directory: string): Promise { const stat = await fs.lstat(directory); @@ -30,7 +31,8 @@ export async function writeEmbeddingJobRecord( name: string, payload: unknown, ): Promise { - if (!RECORD_NAMES.has(name)) throw new Error('Invalid embedding receipt name'); + if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name)) + throw new Error('Invalid embedding receipt name'); const canonical = await embeddingJobDirectory(directory); const serialized = JSON.stringify(payload); if (serialized === undefined) throw new Error('Embedding receipt payload is not JSON'); @@ -56,7 +58,8 @@ export async function writeEmbeddingJobRecord( /** Bounded read; malformed or untrusted receipts never become an empty job. */ export async function readEmbeddingJobRecord(directory: string, name: string): Promise { - if (!RECORD_NAMES.has(name)) throw new Error('Invalid embedding receipt name'); + if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name)) + throw new Error('Invalid embedding receipt name'); const canonical = await embeddingJobDirectory(directory); const target = path.join(canonical, name); if (!(await fs.lstat(target)).isFile()) diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts new file mode 100644 index 000000000..6f2f2ba94 --- /dev/null +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -0,0 +1,186 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type VectorStoreOptions } from '../../src/core/lbug/vector-store.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; + +let temporary: TestDBHandle | undefined; +let store: VectorStore; +const chunk = (chunkIndex: number) => ({ + chunkIndex, + startLine: 1, + endLine: 2, + embedding: [0.1, 0.2, 0.3, 0.4], +}); +async function setup() { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const options: VectorStoreOptions = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'create', + }; + store = await VectorStore.open(options); + return options; +} +afterEach(async () => { + await store?.close(); + store = undefined; + await temporary?.cleanup(); + temporary = undefined; + vi.unstubAllEnvs(); +}); + +describe('separate durable vector artifacts', () => { + it('writes a multi-node unit and verifies a large node in bounded batches', async () => { + const options = await setup(); + const nodes = [0, 1].map((i) => ({ nodeId: `Function:a:node${i}`, contentHash: 'hash' })); + await store.beginNodes(nodes); + for (let start = 0; start < 600; start += 128) { + const rows = Array.from({ length: Math.min(128, 600 - start) }, (_, i) => ({ + ...nodes[start + i < 300 ? 0 : 1], + ...chunk((start + i) % 300), + })); + await store.appendBatch(rows); + } + const receipts = await store.finishNodes(nodes.map((node) => ({ ...node, chunks: 300 }))); + await store.checkpointUnit(receipts); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + for (const receipt of receipts) + expect(await store.reusable(receipt.nodeId, receipt.contentHash)).toEqual(receipt); + expect(await store.seal(2, 600)).toBe('exact'); + }); + + it('publishes an empty artifact without loading a model or ANN extension', async () => { + const options = await setup(); + expect(await store.seal(0, 0)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + }); + + it('replays incomplete groups and removes surplus old chunks without damaging completed groups', async () => { + const options = await setup(); + await store.beginNode('Function:a:complete', 'hash'); + await store.append('Function:a:complete', 'hash', [chunk(0)]); + const complete = await store.finishNode('Function:a:complete', 'hash', 1); + await store.checkpointUnit([complete]); + await store.beginNode('Function:a:active', 'old'); + await store.append('Function:a:active', 'old', [chunk(0), chunk(1), chunk(2)]); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + expect(await store.reusable('Function:a:active', 'old')).toBeNull(); + expect(await store.reusable('Function:a:complete', 'hash')).toEqual(complete); + await store.beginNode('Function:a:active', 'new'); + await store.append('Function:a:active', 'new', [chunk(0)]); + const active = await store.finishNode('Function:a:active', 'new', 1); + await store.checkpointUnit([active]); + expect(await store.seal(2, 2)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.reusable('Function:a:active', 'new')).toEqual(active); + await expect(store.beginNode('Function:a:active', 'other')).rejects.toThrow('immutable'); + }); + + it('rejects invalid batches before mutation and never acknowledges a partial node', async () => { + await setup(); + await store.beginNode('Function:a:node', 'hash'); + for (const invalid of [ + chunk(1), + { ...chunk(0), embedding: [NaN, 0, 0, 0] }, + { ...chunk(0), embedding: [0, 0] }, + { ...chunk(0), endLine: 0 }, + ]) { + await expect(store.append('Function:a:node', 'hash', [invalid])).rejects.toThrow(); + } + await store.append('Function:a:node', 'hash', [chunk(0)]); + await expect( + store.checkpointUnit([ + { nodeId: 'Function:a:node', contentHash: 'hash', chunks: 1, complete: true }, + ]), + ).rejects.toThrow('incomplete'); + await expect(store.finishNode('Function:a:node', 'hash', 2)).rejects.toThrow('coverage'); + await expect(store.seal(1, 1)).rejects.toThrow('incomplete'); + const receipt = await store.finishNode('Function:a:node', 'hash', 1); + await store.checkpointUnit([receipt]); + }); + + it('refuses identity mismatch, missing chunks, and unpublished readers', async () => { + const options = await setup(); + await store.beginNode('Function:a:node', 'hash'); + await store.append('Function:a:node', 'hash', [chunk(0), chunk(1)]); + const receipt = await store.finishNode('Function:a:node', 'hash', 2); + await store.checkpointUnit([receipt]); + await store.close(); + store = undefined; + await expect(VectorStore.open({ ...options, mode: 'read' })).rejects.toThrow('not complete'); + await expect( + VectorStore.open({ ...options, mode: 'resume', identity: 'b'.repeat(64) }), + ).rejects.toThrow('identity'); + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query( + "MATCH (e:CodeEmbedding {id:'Function:a:node:1'}) DELETE e", + ); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } finally { + await closeLbugConnection(handle); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await expect(store.reusable('Function:a:node', 'hash')).rejects.toThrow('missing chunks'); + }); + + it('keeps a separately published graph readable while the vector writer is open', async () => { + const options = await setup(); + const graphPath = path.join(options.directory, 'graph'); + let graph = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const run = async (sql: string) => { + const result = await graph.conn.query(sql); + const parts = Array.isArray(result) ? result : [result]; + try { + return await parts[0].getAll(); + } finally { + parts.forEach((part) => part.close()); + } + }; + await run('CREATE NODE TABLE Function(id STRING, PRIMARY KEY(id))'); + await run("CREATE (:Function {id:'Function:a:node'})"); + await run('CHECKPOINT'); + await closeLbugConnection(graph); + graph = await openLbugConnection(lbug, graphPath, { + readOnly: true, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + await store.beginNode('Function:a:node', 'hash'); + await store.append('Function:a:node', 'hash', [chunk(0)]); + expect(await run('MATCH (n:Function) RETURN n.id AS id')).toEqual([ + { id: 'Function:a:node' }, + ]); + } finally { + await closeLbugConnection(graph); + } + }); + + it('does not replace an existing artifact or accept an unbounded reservation', async () => { + const options = await setup(); + await expect(VectorStore.open(options)).rejects.toThrow('exists'); + await expect(VectorStore.open({ ...options, bufferPoolBytes: 0 })).rejects.toThrow( + 'reservation', + ); + expect((await fs.stat(path.join(options.directory, 'vectors'))).isFile()).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts new file mode 100644 index 000000000..ae0c1bd16 --- /dev/null +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -0,0 +1,261 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + EmbeddingJobJournal, + type EmbeddingUnitIntent, +} from '../../src/storage/embedding-job-journal.js'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; +import type { EmbeddingManifestReceipt } from '../../src/core/embeddings/embedding-work-manifest.js'; +import { embeddingJobIdentity } from '../../src/core/embeddings/embedding-job-identity.js'; + +const identity = 'a'.repeat(64); +const manifest: EmbeddingManifestReceipt = { + version: 1, + digest: 'b'.repeat(64), + sourceIdentity: 'c'.repeat(64), + bytes: 30, + totalNodes: 6, + totalUnits: 3, + maximumUnitBytes: 100, +}; +const unit = (n: number, origin: 'manifest' | 'failed' = 'manifest'): EmbeddingUnitIntent => ({ + start: { unit: n, offset: n * 10 }, + next: { unit: n + 1, offset: (n + 1) * 10 }, + nodes: 2, + origin, +}); +const directories: string[] = []; +async function collect(source: AsyncIterable): Promise { + const items: T[] = []; + for await (const item of source) items.push(item); + return items; +} +async function job() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-journal-')); + directories.push(directory); + return EmbeddingJobJournal.create(directory, identity, manifest); +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { force: true, recursive: true }); +}); + +describe('durable embedding scheduling', () => { + it('resumes only the active unit and keeps acknowledged counts', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.acknowledge({ reusedNodes: 0, chunks: 3 }); + await journal.begin(unit(1)); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ + completedUnits: 1, + completedNodes: 2, + cursor: { unit: 1, offset: 10 }, + active: unit(1), + chunks: 3, + }); + await journal.acknowledge({ reusedNodes: 2, chunks: 4 }); + await journal.begin(unit(2)); + await journal.acknowledge({ reusedNodes: 1, chunks: 2 }); + await journal.indexing(); + await journal.ready(); + expect( + (await EmbeddingJobJournal.open(journal.directory, identity, manifest)).snapshot(), + ).toMatchObject({ + phase: 'ready', + completedUnits: 2, + reusedUnits: 1, + completedNodes: 3, + reusedNodes: 3, + chunks: 9, + active: null, + }); + }); + + it('keeps failures retryable across arbitrarily many invocations', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + expect(await collect(journal.failed())).toEqual([]); + for (let attempt = 0; attempt < 5; attempt++) { + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(await collect(journal.failed())).toEqual([unit(0, 'failed')]); + await journal.begin(unit(0, 'failed')); + expect(journal.snapshot()).toMatchObject({ + failedUnits: 0, + failedNodes: 0, + active: unit(0, 'failed'), + }); + await journal.fail('provider'); + expect(journal.snapshot()).toMatchObject({ + failedUnits: 1, + failedNodes: 2, + cursor: { unit: 1, offset: 10 }, + }); + expect(await collect(journal.failed())).toEqual([]); + } + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + await journal.acknowledge({ reusedNodes: 0, chunks: 2 }); + expect(await collect(journal.failed())).toEqual([]); + expect(journal.snapshot()).toMatchObject({ completedUnits: 1, failedUnits: 0 }); + }); + + it('replays the active failed unit after its record was removed but before cursor acknowledgement', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + await fs.unlink(path.join(journal.directory, 'failed-0000000000000000.json')); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot().active).toEqual(unit(0, 'failed')); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + expect(journal.snapshot()).toMatchObject({ failedUnits: 0, reusedUnits: 1 }); + }); + + it('reconciles an uncertain acknowledgement by reopening instead of counting twice', async () => { + let journal = await job(); + await journal.begin(unit(0)); + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + await rename(from, to); + if (String(to).endsWith('/job.json')) throw new Error('injected failure after rename'); + }); + await expect(journal.acknowledge({ reusedNodes: 0, chunks: 2 })).rejects.toThrow('injected'); + await expect(journal.begin(unit(1))).rejects.toThrow('durability uncertain'); + vi.restoreAllMocks(); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ + completedUnits: 1, + completedNodes: 2, + active: null, + cursor: { unit: 1, offset: 10 }, + }); + await journal.begin(unit(1)); + }); + + it('rejects concurrent mutation instead of queuing unbounded intents', async () => { + const journal = await job(); + const first = journal.begin(unit(0)); + await expect(journal.begin(unit(0))).rejects.toThrow('Concurrent'); + await first; + expect(journal.snapshot().active).toEqual(unit(0)); + }); + + it.each(['identity', 'manifest', 'checksum', 'accounting', 'cursor'])( + 'retains artifacts on corrupt %s', + async (kind) => { + const journal = await job(); + await fs.writeFile(path.join(journal.directory, 'vectors'), 'retained'); + if (kind === 'checksum') + await fs.writeFile( + path.join(journal.directory, 'job.json'), + '{"checksum":"bad","payload":{}}', + ); + if (kind === 'accounting') + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...journal.snapshot(), + completedUnits: 1, + }); + if (kind === 'cursor') + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...journal.snapshot(), + cursor: { unit: 0, offset: 9 }, + }); + await expect( + EmbeddingJobJournal.open( + journal.directory, + kind === 'identity' ? 'f'.repeat(64) : identity, + kind === 'manifest' ? { ...manifest, digest: 'd'.repeat(64) } : manifest, + ), + ).rejects.toThrow(); + expect(await fs.readFile(path.join(journal.directory, 'vectors'), 'utf8')).toBe('retained'); + }, + ); + + it('cannot publish incomplete coverage or mutate the live state through a snapshot', async () => { + const journal = await job(); + journal.snapshot().cursor.unit = 100; + expect(journal.snapshot().cursor.unit).toBe(0); + await expect(journal.indexing()).rejects.toThrow('accounting'); + expect(await readEmbeddingJobRecord(journal.directory, 'job.json')).toMatchObject({ + phase: 'embedding', + }); + }); + + it('validates failure filenames without accepting traversal', async () => { + const journal = await job(); + await expect( + writeEmbeddingJobRecord(journal.directory, '../failed-0000000000000000.json', {}), + ).rejects.toThrow('name'); + await expect(writeEmbeddingJobRecord(journal.directory, 'failed-0.json', {})).rejects.toThrow( + 'name', + ); + }); +}); + +describe('embedding job identity', () => { + const graph = { + generation: 'generation', + coveredContent: 'dirty-coverage', + revision: 'revision', + analyzer: 'analyzer', + parser: 'parser', + schema: 'schema', + }; + const vector = { provider: 'local', model: 'model', dimensions: 384 }; + const config = { + chunkSize: 2000, + overlap: 100, + maxSnippetLength: 2000, + maxDescriptionLength: 500, + maxInputChars: 8192, + }; + it('excludes scheduling options and unrelated credential fields', () => { + const expanded = { + ...config, + batchSize: 1, + threads: 1, + resourceBudget: 1024, + apiKey: 'secret', + }; + expect(embeddingJobIdentity(graph, vector, 'v7', expanded)).toEqual( + embeddingJobIdentity(graph, vector, 'v7', config), + ); + }); + it('invalidates on each coverage and input dimension', () => { + const base = embeddingJobIdentity(graph, vector, 'v7', config); + for (const key of Object.keys(graph)) + expect( + embeddingJobIdentity({ ...graph, [key]: 'changed' }, vector, 'v7', config).identity, + ).not.toBe(base.identity); + for (const key of Object.keys(config)) + expect( + embeddingJobIdentity(graph, vector, 'v7', { + ...config, + [key]: config[key as keyof typeof config] + 1, + }).identity, + ).not.toBe(base.identity); + expect( + embeddingJobIdentity(graph, { ...vector, dimensions: 768 }, 'v7', config).identity, + ).not.toBe(base.identity); + expect( + embeddingJobIdentity(graph, { ...vector, model: 'other' }, 'v7', config).identity, + ).not.toBe(base.identity); + expect(() => + embeddingJobIdentity( + graph, + { ...vector, provider: 'https://user:secret@example.com' }, + 'v7', + config, + ), + ).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/lbug-explicit-budget.test.ts b/gitnexus/test/unit/lbug-explicit-budget.test.ts new file mode 100644 index 000000000..5397e5ff2 --- /dev/null +++ b/gitnexus/test/unit/lbug-explicit-budget.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createLbugDatabase, openLbugConnection } from '../../src/core/lbug/lbug-config.js'; + +afterEach(() => vi.unstubAllEnvs()); + +describe('explicit native handle reservations', () => { + it('uses a per-handle reservation while preserving the environment policy for legacy callers', () => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', '0'); + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database } as unknown as Parameters[0]; + createLbugDatabase(module, '/tmp/budgeted', { bufferPoolBytes: 64 * 1024 * 1024 }); + expect(Database.mock.calls[0][1]).toBe(64 * 1024 * 1024); + createLbugDatabase(module, '/tmp/legacy'); + expect(Database.mock.calls[1][1]).toBe(0); + }); + + it.each([0, -1, NaN, Infinity, 1.5])( + 'rejects invalid explicit reservation %s before opening native', + (bufferPoolBytes) => { + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database } as unknown as Parameters[0]; + expect(() => createLbugDatabase(module, '/tmp/invalid', { bufferPoolBytes })).toThrow( + 'positive byte', + ); + expect(Database).not.toHaveBeenCalled(); + }, + ); + + it('passes an explicit thread count only when requested', async () => { + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const Connection = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database, Connection } as unknown as Parameters[0]; + await openLbugConnection(module, '/tmp/threads', { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 2, + }); + expect(Connection.mock.calls[0]).toHaveLength(2); + expect(Connection.mock.calls[0][1]).toBe(2); + await openLbugConnection(module, '/tmp/legacy'); + expect(Connection.mock.calls[1]).toHaveLength(1); + await expect(openLbugConnection(module, '/tmp/invalid', { threads: 0 })).rejects.toThrow( + 'positive integer', + ); + expect(Database).toHaveBeenCalledTimes(2); + }); +}); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 684046e6e..f2ac1bbd6 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -1,6 +1,9 @@ import { defineConfig } from 'vitest/config'; import PerfSequencer from './test/helpers/perf-sequencer.js'; +// Keep native tests in exactly one project (TESTING.md). +const EMBEDDING_VECTOR_STORE_TEST = 'test/integration/embedding-vector-store.test.ts'; + export default defineConfig({ test: { // Shared settings — inherited by all projects via extends: true @@ -78,6 +81,7 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + EMBEDDING_VECTOR_STORE_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', @@ -196,6 +200,7 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + EMBEDDING_VECTOR_STORE_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', From 3eff18478d782459f092059c5516e1c8add8c8a9 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 12:19:22 +0000 Subject: [PATCH 05/67] feat(embeddings): stream resumable job units --- gitnexus/bench/analyze-resume/README.md | 14 + .../text-streaming-baseline.json | 440 ++++++++++++++++++ .../bench/analyze-resume/text-streaming.mjs | 102 ++++ .../src/core/embeddings/character-chunk.ts | 90 ++-- gitnexus/src/core/embeddings/chunker.ts | 167 ++++--- .../core/embeddings/embedding-job-runner.ts | 382 +++++++++++++++ .../src/core/embeddings/embedding-pipeline.ts | 7 +- gitnexus/src/core/embeddings/line-index.ts | 14 +- .../src/core/embeddings/text-generator.ts | 82 +++- gitnexus/src/core/lbug/embedding-source.ts | 181 +++++++ gitnexus/src/core/lbug/vector-store.ts | 107 ++++- gitnexus/src/storage/embedding-job-journal.ts | 58 +++ .../embedding-vector-store.test.ts | 381 +++++++++++++++ .../unit/embedding-streaming-text.test.ts | 86 ++++ 14 files changed, 1980 insertions(+), 131 deletions(-) create mode 100644 gitnexus/bench/analyze-resume/text-streaming-baseline.json create mode 100644 gitnexus/bench/analyze-resume/text-streaming.mjs create mode 100644 gitnexus/src/core/embeddings/embedding-job-runner.ts create mode 100644 gitnexus/src/core/lbug/embedding-source.ts create mode 100644 gitnexus/test/unit/embedding-streaming-text.test.ts diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index e60a9a6e2..18838031d 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -40,3 +40,17 @@ both paths. This isolates write overhead; it does not measure inference or graph ingestion. `--check` rejects a candidate median above the baseline median. The initial Linux/Node 22 run in `vector-store-baseline.json` measured a 0.470 candidate/baseline median ratio, including the durability journal. + +Compare text preparation and streaming chunks with an unchanged built checkout: + +```sh +node --expose-gc bench/analyze-resume/text-streaming.mjs /path/to/baseline/gitnexus/dist +``` + +The probe warms both implementations, alternates seven measured pairs, and checks +identical content hashes and every chunk's text, offsets, and line spans. The +baseline checkout for `text-streaming-baseline.json` was commit +`f641342d1fcbfdcb764c5e478db0f69e6cea899a`. Candidate/baseline median ratios were +0.957 for ordinary snippets, 0.948 for many short lines, 0.917 for an oversized +short-label node, 0.919 for a large declaration, and 0.595 for character chunks. +These are preparation costs, not end-to-end inference or analysis measurements. diff --git a/gitnexus/bench/analyze-resume/text-streaming-baseline.json b/gitnexus/bench/analyze-resume/text-streaming-baseline.json new file mode 100644 index 000000000..020d4ce7f --- /dev/null +++ b/gitnexus/bench/analyze-resume/text-streaming-baseline.json @@ -0,0 +1,440 @@ +{ + "node": "v22.23.3", + "platform": "linux", + "fingerprints": { + "short-code": "6f3ccc2ffd9833584b144a688e0d711911ba506c", + "many-short-lines": "b903948f55a0e163eeed5d3f1dae1a3ada34b011", + "large-short-label": "f10a16214a869c4aa62fb0afc926bae99db60fef", + "large-declaration": "ce8401d3b4df5cc25d8378323ff056ca101d04b2", + "chunks": "78e8ab79f72aeea64cd8b1aaa3aa50c766cb1a7cf6402e61a72d3f81cca496e8" + }, + "ratios": { + "short-code": 0.9570562752783565, + "many-short-lines": 0.9482353771383033, + "large-short-label": 0.9167695057681882, + "large-declaration": 0.9190759176146885, + "chunks": 0.5951500187087568 + }, + "samples": [ + { + "case": "short-code", + "trial": 0, + "mode": "baseline", + "ms": 15.460057000000234 + }, + { + "case": "short-code", + "trial": 0, + "mode": "candidate", + "ms": 14.650719000001118 + }, + { + "case": "short-code", + "trial": 1, + "mode": "candidate", + "ms": 16.895096999998714 + }, + { + "case": "short-code", + "trial": 1, + "mode": "baseline", + "ms": 16.1013669999993 + }, + { + "case": "short-code", + "trial": 2, + "mode": "baseline", + "ms": 15.971506999998383 + }, + { + "case": "short-code", + "trial": 2, + "mode": "candidate", + "ms": 15.285631000000649 + }, + { + "case": "short-code", + "trial": 3, + "mode": "candidate", + "ms": 16.652016000000003 + }, + { + "case": "short-code", + "trial": 3, + "mode": "baseline", + "ms": 15.748994000001403 + }, + { + "case": "short-code", + "trial": 4, + "mode": "baseline", + "ms": 19.216942000000927 + }, + { + "case": "short-code", + "trial": 4, + "mode": "candidate", + "ms": 15.05308899999909 + }, + { + "case": "short-code", + "trial": 5, + "mode": "candidate", + "ms": 14.682182999998986 + }, + { + "case": "short-code", + "trial": 5, + "mode": "baseline", + "ms": 18.089461999999912 + }, + { + "case": "short-code", + "trial": 6, + "mode": "baseline", + "ms": 14.437770000000455 + }, + { + "case": "short-code", + "trial": 6, + "mode": "candidate", + "ms": 15.846268000001146 + }, + { + "case": "many-short-lines", + "trial": 0, + "mode": "baseline", + "ms": 222.07076000000052 + }, + { + "case": "many-short-lines", + "trial": 0, + "mode": "candidate", + "ms": 204.5821530000012 + }, + { + "case": "many-short-lines", + "trial": 1, + "mode": "candidate", + "ms": 199.73077399999966 + }, + { + "case": "many-short-lines", + "trial": 1, + "mode": "baseline", + "ms": 203.6863659999999 + }, + { + "case": "many-short-lines", + "trial": 2, + "mode": "baseline", + "ms": 203.15121700000054 + }, + { + "case": "many-short-lines", + "trial": 2, + "mode": "candidate", + "ms": 194.02467500000057 + }, + { + "case": "many-short-lines", + "trial": 3, + "mode": "candidate", + "ms": 200.72532900000078 + }, + { + "case": "many-short-lines", + "trial": 3, + "mode": "baseline", + "ms": 208.93676400000004 + }, + { + "case": "many-short-lines", + "trial": 4, + "mode": "baseline", + "ms": 215.45779399999992 + }, + { + "case": "many-short-lines", + "trial": 4, + "mode": "candidate", + "ms": 187.69042199999967 + }, + { + "case": "many-short-lines", + "trial": 5, + "mode": "candidate", + "ms": 194.5716599999996 + }, + { + "case": "many-short-lines", + "trial": 5, + "mode": "baseline", + "ms": 205.19342000000142 + }, + { + "case": "many-short-lines", + "trial": 6, + "mode": "baseline", + "ms": 200.889713999999 + }, + { + "case": "many-short-lines", + "trial": 6, + "mode": "candidate", + "ms": 193.9382970000006 + }, + { + "case": "large-short-label", + "trial": 0, + "mode": "baseline", + "ms": 8.832861999999295 + }, + { + "case": "large-short-label", + "trial": 0, + "mode": "candidate", + "ms": 7.626137999999628 + }, + { + "case": "large-short-label", + "trial": 1, + "mode": "candidate", + "ms": 8.024664000000485 + }, + { + "case": "large-short-label", + "trial": 1, + "mode": "baseline", + "ms": 8.374761000000944 + }, + { + "case": "large-short-label", + "trial": 2, + "mode": "baseline", + "ms": 8.299746999999115 + }, + { + "case": "large-short-label", + "trial": 2, + "mode": "candidate", + "ms": 7.570765000000392 + }, + { + "case": "large-short-label", + "trial": 3, + "mode": "candidate", + "ms": 7.739359000001059 + }, + { + "case": "large-short-label", + "trial": 3, + "mode": "baseline", + "ms": 8.44199000000117 + }, + { + "case": "large-short-label", + "trial": 4, + "mode": "baseline", + "ms": 8.244589999998425 + }, + { + "case": "large-short-label", + "trial": 4, + "mode": "candidate", + "ms": 7.600515000000087 + }, + { + "case": "large-short-label", + "trial": 5, + "mode": "candidate", + "ms": 8.456828999998834 + }, + { + "case": "large-short-label", + "trial": 5, + "mode": "baseline", + "ms": 8.454603000000134 + }, + { + "case": "large-short-label", + "trial": 6, + "mode": "baseline", + "ms": 8.904308999999557 + }, + { + "case": "large-short-label", + "trial": 6, + "mode": "candidate", + "ms": 7.965303999999378 + }, + { + "case": "large-declaration", + "trial": 0, + "mode": "baseline", + "ms": 43.31495399999949 + }, + { + "case": "large-declaration", + "trial": 0, + "mode": "candidate", + "ms": 35.47860899999978 + }, + { + "case": "large-declaration", + "trial": 1, + "mode": "candidate", + "ms": 35.414584000000104 + }, + { + "case": "large-declaration", + "trial": 1, + "mode": "baseline", + "ms": 46.11522800000057 + }, + { + "case": "large-declaration", + "trial": 2, + "mode": "baseline", + "ms": 36.52931399999943 + }, + { + "case": "large-declaration", + "trial": 2, + "mode": "candidate", + "ms": 34.8059489999996 + }, + { + "case": "large-declaration", + "trial": 3, + "mode": "candidate", + "ms": 35.28219600000011 + }, + { + "case": "large-declaration", + "trial": 3, + "mode": "baseline", + "ms": 38.20857099999921 + }, + { + "case": "large-declaration", + "trial": 4, + "mode": "baseline", + "ms": 38.03440099999898 + }, + { + "case": "large-declaration", + "trial": 4, + "mode": "candidate", + "ms": 34.95650199999909 + }, + { + "case": "large-declaration", + "trial": 5, + "mode": "candidate", + "ms": 34.83934399999998 + }, + { + "case": "large-declaration", + "trial": 5, + "mode": "baseline", + "ms": 37.985978000000614 + }, + { + "case": "large-declaration", + "trial": 6, + "mode": "baseline", + "ms": 37.7010850000006 + }, + { + "case": "large-declaration", + "trial": 6, + "mode": "candidate", + "ms": 34.53734900000018 + }, + { + "case": "chunks", + "trial": 0, + "mode": "baseline", + "ms": 65.19520499999999 + }, + { + "case": "chunks", + "trial": 0, + "mode": "candidate", + "ms": 39.78473800000029 + }, + { + "case": "chunks", + "trial": 1, + "mode": "candidate", + "ms": 37.32188899999892 + }, + { + "case": "chunks", + "trial": 1, + "mode": "baseline", + "ms": 67.45899199999985 + }, + { + "case": "chunks", + "trial": 2, + "mode": "baseline", + "ms": 62.80641800000012 + }, + { + "case": "chunks", + "trial": 2, + "mode": "candidate", + "ms": 39.00661599999876 + }, + { + "case": "chunks", + "trial": 3, + "mode": "candidate", + "ms": 39.36458599999969 + }, + { + "case": "chunks", + "trial": 3, + "mode": "baseline", + "ms": 64.03954599999997 + }, + { + "case": "chunks", + "trial": 4, + "mode": "baseline", + "ms": 65.2007450000001 + }, + { + "case": "chunks", + "trial": 4, + "mode": "candidate", + "ms": 37.91183699999965 + }, + { + "case": "chunks", + "trial": 5, + "mode": "candidate", + "ms": 37.205321999999796 + }, + { + "case": "chunks", + "trial": 5, + "mode": "baseline", + "ms": 62.19722600000023 + }, + { + "case": "chunks", + "trial": 6, + "mode": "baseline", + "ms": 63.657884000000195 + }, + { + "case": "chunks", + "trial": 6, + "mode": "candidate", + "ms": 38.11313700000028 + } + ] +} diff --git a/gitnexus/bench/analyze-resume/text-streaming.mjs b/gitnexus/bench/analyze-resume/text-streaming.mjs new file mode 100644 index 000000000..70f9f2bfe --- /dev/null +++ b/gitnexus/bench/analyze-resume/text-streaming.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +// Compare identical text and chunk outputs against an unchanged built checkout. +// node --expose-gc bench/analyze-resume/text-streaming.mjs /path/to/baseline/gitnexus/dist +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createHash } from 'node:crypto'; +import * as candidateText from '../../dist/core/embeddings/embedding-pipeline.js'; +import * as candidateChunks from '../../dist/core/embeddings/character-chunk.js'; + +if (!process.argv[2]) throw new Error('Pass the unchanged baseline dist directory'); +const baselineDirectory = path.resolve(process.argv[2]); +const baselineText = await import( + pathToFileURL(path.join(baselineDirectory, 'core/embeddings/embedding-pipeline.js')) +); +const baselineChunks = await import( + pathToFileURL(path.join(baselineDirectory, 'core/embeddings/character-chunk.js')) +); +const node = (label, content) => ({ + id: `${label}:source.ts:Example`, + name: 'Example', + filePath: 'source.ts', + label, + content, + startLine: 1, + endLine: 1, +}); +const cases = [ + { + name: 'short-code', + node: node('Function', 'export function example() { return "漢😀"; }\n'), + repetitions: 10000, + }, + { name: 'many-short-lines', node: node('File', 'a \t\r\n\n\n'.repeat(600000)), repetitions: 1 }, + { + name: 'large-short-label', + node: node('TypeAlias', 'type Huge = ' + '"漢😀" | '.repeat(300000) + 'never;\n'), + repetitions: 1, + }, + { + name: 'large-declaration', + node: node( + 'Class', + 'export class Example {\n' + + ' field: string;\n'.repeat(100000) + + ' method() { return "😀"; }\n}\n', + ), + repetitions: 1, + }, +]; +const samples = []; +const fingerprints = {}; +for (const entry of cases) { + for (let trial = -1; trial < 7; trial++) { + for (const mode of trial % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { + global.gc?.(); + const impl = mode === 'candidate' ? candidateText : baselineText; + const start = performance.now(); + let fingerprint; + for (let i = 0; i < entry.repetitions; i++) fingerprint = impl.contentHashForNode(entry.node); + const ms = performance.now() - start; + if (fingerprints[entry.name]) assert.equal(fingerprint, fingerprints[entry.name]); + fingerprints[entry.name] = fingerprint; + if (trial >= 0) samples.push({ case: entry.name, trial, mode, ms }); + } + } +} +const content = 'a\n'.repeat(2 * 1024 * 1024); +for (let trial = -1; trial < 7; trial++) { + for (const mode of trial % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { + global.gc?.(); + const impl = mode === 'candidate' ? candidateChunks : baselineChunks; + const start = performance.now(); + const chunks = + mode === 'candidate' + ? impl.iterateCharacterChunks(content, 1, 2097153, 1200, 120) + : impl.characterChunk(content, 1, 2097153, 1200, 120); + const hash = createHash('sha256'); + for (const chunk of chunks) hash.update(JSON.stringify(chunk)); + const ms = performance.now() - start; + const fingerprint = hash.digest('hex'); + if (fingerprints.chunks) assert.equal(fingerprint, fingerprints.chunks); + fingerprints.chunks = fingerprint; + if (trial >= 0) samples.push({ case: 'chunks', trial, mode, ms }); + } +} +const ratios = {}; +for (const name of Object.keys(fingerprints)) { + const median = (mode) => + samples + .filter((s) => s.case === name && s.mode === mode) + .map((s) => s.ms) + .sort((a, b) => a - b)[3]; + ratios[name] = median('candidate') / median('baseline'); +} +process.stdout.write( + JSON.stringify( + { node: process.version, platform: process.platform, fingerprints, ratios, samples }, + null, + 2, + ) + '\n', +); diff --git a/gitnexus/src/core/embeddings/character-chunk.ts b/gitnexus/src/core/embeddings/character-chunk.ts index e4f6f949c..0d0743264 100644 --- a/gitnexus/src/core/embeddings/character-chunk.ts +++ b/gitnexus/src/core/embeddings/character-chunk.ts @@ -2,8 +2,6 @@ * Character-based sliding window chunking (pure, no tree-sitter dependency) */ -import { buildLineIndex, resolveChunkLines } from './line-index.js'; - export interface Chunk { text: string; chunkIndex: number; @@ -13,14 +11,66 @@ export interface Chunk { endLine: number; } +/** Sliding windows with constant auxiliary memory, including line tracking. */ +export function* iterateCharacterChunks( + content: string, + startLine: number, + endLine: number, + chunkSize = 1200, + overlap = 120, +): Generator { + if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || !Number.isSafeInteger(overlap)) + throw new Error('Invalid character chunk size or overlap'); + if (content.length <= chunkSize) { + yield { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }; + return; + } + let offset = 0; + let chunkIndex = 0; + let relativeStart = 0; + let relativeEnd = 0; + let nextStartNewline = content.indexOf('\n'); + let nextEndNewline = nextStartNewline; + while (offset < content.length) { + const end = Math.min(offset + chunkSize, content.length); + while (nextStartNewline >= 0 && nextStartNewline < offset) { + relativeStart++; + nextStartNewline = content.indexOf('\n', nextStartNewline + 1); + } + while (nextEndNewline >= 0 && nextEndNewline < end - 1) { + relativeEnd++; + nextEndNewline = content.indexOf('\n', nextEndNewline + 1); + } + yield { + text: content.slice(offset, end), + chunkIndex: chunkIndex++, + startOffset: offset, + endOffset: end, + startLine: startLine + relativeStart, + endLine: startLine + relativeEnd, + }; + offset = end - overlap; + if (offset >= content.length || end >= content.length) break; + if (offset <= (chunkIndex > 1 ? end - chunkSize : 0)) offset = end; + } +} + +/** Compatibility API for callers that need all chunks. */ export const characterChunk = ( content: string, startLine: number, endLine: number, - chunkSize: number = 1200, - overlap: number = 120, + chunkSize = 1200, + overlap = 120, ): Chunk[] => { - if (content.length <= chunkSize) { + if (content.length <= chunkSize) return [ { text: content, @@ -31,33 +81,5 @@ export const characterChunk = ( endLine, }, ]; - } - - const chunks: Chunk[] = []; - let offset = 0; - const lineOffsets = buildLineIndex(content); - - while (offset < content.length) { - const end = Math.min(offset + chunkSize, content.length); - const chunkText = content.slice(offset, end); - const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine); - - chunks.push({ - text: chunkText, - chunkIndex: chunks.length, - startOffset: offset, - endOffset: end, - startLine: lineRange.startLine, - endLine: lineRange.endLine, - }); - - offset = end - overlap; - if (offset >= content.length) break; - if (end >= content.length) break; - if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) { - offset = end; - } - } - - return chunks; + return [...iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap)]; }; diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts index f3e64aaea..5bef87ebb 100644 --- a/gitnexus/src/core/embeddings/chunker.ts +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -9,7 +9,7 @@ export { type Chunk, characterChunk } from './character-chunk.js'; -import { characterChunk } from './character-chunk.js'; +import { characterChunk, iterateCharacterChunks } from './character-chunk.js'; import type { Chunk } from './character-chunk.js'; import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js'; import { buildLineIndex, resolveChunkLines } from './line-index.js'; @@ -81,10 +81,68 @@ export const chunkNode = async ( filePath: string, startLine: number, endLine: number, - chunkSize: number = 1200, - overlap: number = 120, + chunkSize = 1200, + overlap = 120, options?: ChunkOptions, ): Promise => { + if (label !== 'Class' && content.length <= chunkSize) + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + return [ + ...(await prepareNodeChunks( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + options, + )), + ]; +}; + +/** Produce one chunk at a time, without retaining generated text for the node. */ +export async function* iterateNodeChunks( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize = 1200, + overlap = 120, + options?: ChunkOptions, +): AsyncGenerator { + yield* await prepareNodeChunks( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + options, + ); +} + +async function prepareNodeChunks( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, + options?: ChunkOptions, +): Promise> { // Even a short class contains implementations. Never fall back to raw Class // source: an unavailable parser yields identity/name metadata only. if (label === 'Class') { @@ -105,28 +163,17 @@ export const chunkNode = async ( return [classMetadataChunk(startLine, endLine)]; } } - // Content fits in one chunk — no splitting needed if (content.length <= chunkSize) { - return [ - { - text: content, - chunkIndex: 0, - startOffset: 0, - endOffset: content.length, - startLine, - endLine, - }, - ]; + return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); } - const rule = CHUNKING_RULES[label]; if (!rule || options?.strategy === 'fixed') { - return characterChunk(content, startLine, endLine, chunkSize, overlap); + return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); } - + let chunks: Generator | undefined; try { - if (rule.mode === CHUNK_MODE_AST_FUNCTION) { - const astChunks = await astChunk( + if (rule?.mode === CHUNK_MODE_AST_FUNCTION) { + chunks = await astChunk( content, filePath, startLine, @@ -136,11 +183,8 @@ export const chunkNode = async ( rule, options?.name, ); - if (astChunks.length > 0) return astChunks; - } - - if (rule.mode === CHUNK_MODE_AST_DECLARATION) { - const declarationChunks = await declarationChunk( + } else if (rule?.mode === CHUNK_MODE_AST_DECLARATION) { + chunks = await declarationChunk( content, filePath, startLine, @@ -149,15 +193,13 @@ export const chunkNode = async ( overlap, rule, ); - if (declarationChunks.length > 0) return declarationChunks; } } catch { - // AST parsing failed — fall through to character fallback + // Only preparation may fall back. Never restart after yielding a chunk: + // the consumer may already have durably stored it. } - - // Character-based fallback for everything else - return characterChunk(content, startLine, endLine, chunkSize, overlap); -}; + return chunks ?? iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); +} /** * AST-based chunking for Function/Method @@ -173,9 +215,9 @@ const astChunk = async ( overlap: number, rule: ChunkingRule, expectedName?: string, -): Promise => { +): Promise | undefined> => { const tree = await ensureAndParse(content, filePath); - if (!tree) return []; + if (!tree) return undefined; const root = tree.rootNode; const lineOffsets = buildLineIndex(content); @@ -183,11 +225,11 @@ const astChunk = async ( // Find the function/method declaration in the snippet AST. // tree-sitter parses node.content (a snippet), so rows are relative (0-based). const targetNode = findFunctionNode(root, expectedName); - if (!targetNode) return []; + if (!targetNode) return undefined; // Get the body (statements) via childForFieldName('body') const bodyNode = targetNode.childForFieldName('body'); - if (!bodyNode) return []; + if (!bodyNode) return undefined; // Extract individual statements const statements: Array<{ startIndex: number; endIndex: number }> = []; @@ -208,7 +250,7 @@ const astChunk = async ( } } - if (statements.length === 0) return []; + if (statements.length === 0) return undefined; // A fitting statement must not fall into character splitting just because // its function header or closing delimiters consume the remaining budget. @@ -397,18 +439,18 @@ const declarationChunk = async ( chunkSize: number, overlap: number, rule: ChunkingRule, -): Promise => { +): Promise | undefined> => { const tree = await ensureAndParse(content, filePath); - if (!tree) return []; + if (!tree) return undefined; const targetNode = findDeclarationNode(tree.rootNode); - if (!targetNode) return []; + if (!targetNode) return undefined; const bodyNode = getDeclarationBodyNode(targetNode); - if (!bodyNode) return []; + if (!bodyNode) return undefined; const members = collectDeclarationUnits(bodyNode, rule.groupFields); - if (members.length === 0) return []; + if (members.length === 0) return undefined; return chunkByUnits( content, @@ -443,7 +485,7 @@ const buildChunk = ( }; }; -const chunkByUnits = ( +function* chunkByUnits( content: string, lineOffsets: Int32Array, baseStartLine: number, @@ -454,8 +496,8 @@ const chunkByUnits = ( containerEndOffset: number, includeContainerPrefixOnFirstChunk: boolean, includeContainerSuffixOnLastChunk: boolean, -): Chunk[] => { - const chunks: Chunk[] = []; +): Generator { + let chunkIndex = 0; let chunkStartUnitIdx = 0; while (chunkStartUnitIdx < units.length) { @@ -496,32 +538,31 @@ const chunkByUnits = ( oversizedEndOffset, baseStartLine, ); - const oversizedChunks = characterChunk( + for (const chunk of iterateCharacterChunks( content.slice(oversizedStartOffset, oversizedEndOffset), oversizedLineRange.startLine, oversizedLineRange.endLine, chunkSize, overlap, - ).map((chunk, offsetIdx) => ({ - ...chunk, - chunkIndex: chunks.length + offsetIdx, - startOffset: chunk.startOffset + oversizedStartOffset, - endOffset: chunk.endOffset + oversizedStartOffset, - })); - chunks.push(...oversizedChunks); + )) { + yield { + ...chunk, + chunkIndex: chunkIndex++, + startOffset: chunk.startOffset + oversizedStartOffset, + endOffset: chunk.endOffset + oversizedStartOffset, + }; + } chunkStartUnitIdx += 1; continue; } - chunks.push( - buildChunk( - content, - lineOffsets, - chunks.length, - chunkStartOffset, - candidateEndOffset, - baseStartLine, - ), + yield buildChunk( + content, + lineOffsets, + chunkIndex++, + chunkStartOffset, + candidateEndOffset, + baseStartLine, ); if (chunkEndUnitIdx === units.length - 1) { @@ -540,9 +581,7 @@ const chunkByUnits = ( chunkStartUnitIdx = nextChunkStartUnitIdx; } } - - return chunks; -}; +} const findOverlapStartIndex = ( statements: Array<{ startIndex: number; endIndex: number }>, @@ -772,7 +811,7 @@ const projectClassChunks = ( segment.source + Math.max(0, Math.min(offset - segment.start, segment.end - segment.start)) ); }; - return chunks.map((chunk) => { + return Array.from(chunks, (chunk) => { const startOffset = sourceOffset(chunk.startOffset); const endOffset = sourceOffset(chunk.endOffset - 1) + 1; return { diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts new file mode 100644 index 000000000..fd0ab7cd3 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -0,0 +1,382 @@ +/** One durable manifest unit, one bounded provider batch, no corpus-sized maps. */ +import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; +import type { VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; +import type { + EmbeddingJobJournal, + EmbeddingJobState, + EmbeddingUnitIntent, +} from '../../storage/embedding-job-journal.js'; +import { + fetchEmbeddingWorkUnit, + readEmbeddingWorkUnits, + type EmbeddingManifestReceipt, + type EmbeddingManifestSource, + type EmbeddingWorkUnit, +} from './embedding-work-manifest.js'; +import { contentHashForNode, embedInputRanges, isInputRejection } from './embedding-pipeline.js'; +import { iterateNodeChunks } from './chunker.js'; +import { generateEmbeddingText, splitEmbeddingText } from './text-generator.js'; +import { extractStructuralNames } from './structural-extractor.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { isShortLabel, STRUCTURAL_LABELS, type EmbeddingConfig } from './types.js'; +import type { embedBatch } from './embedder.js'; + +export interface EmbeddingJobRunnerOptions { + directory: string; + manifest: EmbeddingManifestReceipt; + source: EmbeddingManifestSource; + journal: EmbeddingJobJournal; + store: VectorStore; + /** The controller has validated vector/input identities before providing reuse. */ + reuse?: Pick; + config?: Partial; + queueBytes: number; + provider: typeof embedBatch; + initializeProvider?: () => Promise; + signal?: AbortSignal; + onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void; +} + +export interface EmbeddingJobRunResult { + state: EmbeddingJobState; + peakQueueBytes: number; + /** Absent while failed work remains. Publication is the controller's job. */ + indexKind?: 'hnsw' | 'exact'; +} + +/** Caller owns the job lock and closes all handles, including on rejection. */ +export async function runEmbeddingJob( + options: EmbeddingJobRunnerOptions, +): Promise { + const { directory, manifest, source, journal, store, signal } = options; + const config = resolveEmbeddingConfig(options.config ?? {}); + const budget = new EmbeddingByteBudget(options.queueBytes); + const dimensions = store.options.dimensions; + // UTF-16 text + worst-case JSON escaping/encoding + float response, JS array + // and native copies. The provider's runtime has a separate reservation. + const rowBytes = (textChars: number, idChars: number) => + 16 * textChars + 32 * dimensions + 1024 + 4 * idChars; + const minimumBatchBytes = rowBytes(config.maxInputChars, 0); + if (minimumBatchBytes > budget.limit) + throw new Error('Embedding input/vector batch does not fit the resource budget'); + let initialized = false; + let attempted = 0; + let failures = 0; + let consecutiveFailures = 0; + const remainingNodes = + manifest.totalNodes - journal.snapshot().completedNodes - journal.snapshot().reusedNodes; + const ratioFloor = Math.min(20, Math.max(5, Math.ceil(remainingNodes / config.subBatchSize / 2))); + // Bounded even if every source node is rejected. IDs are needed only until + // the existing five-node input-rejection guard fires or a success resets it. + const rejected = new Set(); + const progress = () => options.onProgress?.(journal.snapshot(), budget.peakBytes); + + const processUnit = async ( + unit: EmbeddingWorkUnit, + intent: EmbeddingUnitIntent, + ): Promise => { + signal?.throwIfAborted(); + const complete = await store.completedNodes(unit.nodes.map((node) => node.id)); + const missing = unit.nodes.filter((node) => !complete.has(node.id)); + const retainedBytes = missing.reduce( + (n, key) => n + 2 * (key.contentBytes + key.metadataBytes), + 0, + ); + // One node may expand its text/AST while the rest of the fetched unit stays + // retained. Admission precedes intent, provider calls and vector mutation. + const scratchBytes = missing.reduce( + (n, key) => Math.max(n, 16 * (key.contentBytes + key.metadataBytes)), + 0, + ); + const maxIdChars = missing.reduce((n, key) => Math.max(n, key.id.length), 0); + const maximumBatchRows = Math.min( + 256, + config.subBatchSize, + Math.floor(store.options.batchBytes / (16 * dimensions + 128 + 2 * (maxIdChars + 40))), + ); + if (maximumBatchRows < 1) throw new Error('Vector write does not fit its batch reservation'); + if (retainedBytes + scratchBytes + rowBytes(config.maxInputChars, maxIdChars) > budget.limit) + throw new Error( + `Embedding unit ${unit.ordinal} requires a larger resource budget; no source text was truncated`, + ); + if (!journal.snapshot().active) await journal.begin(intent); + progress(); + if (!missing.length) { + await store.checkpointUnit([...complete.values()]); + await journal.acknowledge({ + reusedNodes: complete.size, + chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), + }); + progress(); + return; + } + const fetched = await fetchEmbeddingWorkUnit( + source, + { ...unit, nodes: missing, retainedBytes }, + budget, + ); + const releaseScratch = budget.reserve(scratchBytes); + let batch: Array & { text: string; release: () => void }> = + []; + const generated = new Map(); + const failed = new Set(); + let failureCause: 'input' | 'provider' = 'input'; + let abort: unknown; + let reusedNodes = complete.size; + + const flush = async (): Promise => { + if (!batch.length) return; + signal?.throwIfAborted(); + if (!initialized) { + await options.initializeProvider?.(); + initialized = true; + } + attempted++; + let providerFailed = false; + try { + for await (const result of embedInputRanges( + batch.map((row) => row.text), + batch.length, + signal, + options.provider, + )) { + signal?.throwIfAborted(); + const rows = batch.slice(result.start, result.end); + if ('error' in result) { + for (const row of rows) failed.add(row.nodeId); + if (isInputRejection(result.error)) { + for (const row of rows) if (rejected.size < 5) rejected.add(row.nodeId); + } else { + failureCause = 'provider'; + providerFailed = true; + consecutiveFailures++; + if (consecutiveFailures >= 5) abort = result.error; + } + continue; + } + if ( + result.embeddings.length !== rows.length || + result.embeddings.some( + (v) => v.length !== dimensions || v.some((n) => !Number.isFinite(n)), + ) + ) + throw new Error('Embedding provider returned invalid vector coverage'); + consecutiveFailures = 0; + rejected.clear(); + const updates = rows.flatMap((row, i) => + failed.has(row.nodeId) + ? [] + : [ + { + nodeId: row.nodeId, + contentHash: row.contentHash, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + embedding: Array.from(result.embeddings[i]), + }, + ], + ); + if (updates.length) await store.appendBatch(updates); + } + if (providerFailed) failures++; + if (!abort && providerFailed && attempted >= ratioFloor && failures / attempted >= 0.25) + abort = new Error( + 'Embedding endpoint failure rate exceeded 25%; unfinished units remain resumable', + ); + if (!abort && rejected.size >= 5) + abort = new Error( + 'Embedding inputs rejected for five nodes without a success; unfinished units remain resumable', + ); + } finally { + for (const row of batch) row.release(); + batch = []; + } + }; + + try { + // Hash once per fetched node; current-job complete groups never fetch/hash. + const inputs = fetched.nodes.map((node) => ({ + nodeId: node.id, + contentHash: contentHashForNode(node, config), + sourceStartLine: node.startLine ?? 0, + sourceEndLine: node.endLine ?? 0, + })); + await store.beginNodes(inputs); + for (let i = 0; i < fetched.nodes.length; i++) { + const node = fetched.nodes[i]; + const input = inputs[i]; + signal?.throwIfAborted(); + const reuse = options.reuse; + const reusable = await reuse?.reusable(node.id, input.contentHash); + if (reusable && reuse) { + // Flush before copy so there is only one vector batch in memory. + await flush(); + if (abort) break; + const copyRows = Math.min( + 256, + Math.floor( + Math.min( + reuse.options.batchBytes, + store.options.batchBytes, + budget.limit - budget.usedBytes, + ) / rowBytes(0, node.id.length), + ), + ); + if (copyRows < 1) + throw new Error('Reusable vector batch does not fit the resource budget'); + for (let start = 0; start < reusable.chunks; start += copyRows) { + const length = Math.min(copyRows, reusable.chunks - start); + const release = budget.reserve(length * rowBytes(0, node.id.length)); + try { + const chunks = await reuse.readChunks(reusable, start, length); + const delta = input.sourceStartLine - (reusable.sourceStartLine ?? 0); + await store.append( + node.id, + input.contentHash, + chunks.map((chunk) => ({ + ...chunk, + startLine: chunk.startLine + delta, + endLine: + chunk.endLine === reusable.sourceEndLine + ? input.sourceEndLine + : chunk.endLine + delta, + })), + ); + } finally { + release(); + } + } + complete.set( + node.id, + await store.finishNode(node.id, input.contentHash, reusable.chunks), + ); + reusedNodes++; + continue; + } + if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames(node.content, node.filePath); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + /* Same optional structural enrichment as the legacy pipeline. */ + } + } + const state = { contentHash: input.contentHash, chunks: 0, ended: false }; + generated.set(node.id, state); + const chunks = isShortLabel(node.label) + ? [ + { + text: node.content, + chunkIndex: 0, + startLine: node.startLine ?? 0, + endLine: node.endLine ?? 0, + }, + ] + : iterateNodeChunks( + node.label, + node.content, + node.filePath, + node.startLine ?? 0, + node.endLine ?? 0, + config.chunkSize, + config.overlap, + ); + let prevTail = ''; + for await (const chunk of chunks) { + if (failed.has(node.id) || abort) break; + const text = generateEmbeddingText(node, chunk.text, config, chunk.chunkIndex, prevTail); + for (const inputText of splitEmbeddingText(text, config.maxInputChars)) { + const bytes = rowBytes(inputText.length, node.id.length); + if ( + batch.length && + (batch.length >= maximumBatchRows || budget.usedBytes + bytes > budget.limit) + ) + await flush(); + if (failed.has(node.id) || abort) break; + const release = budget.reserve(bytes); + // A sliced string can pin an entire oversized generated declaration. + // UTF-16 detachment preserves even unpaired surrogate code units. + const detached = + text.length > config.maxInputChars + ? Buffer.from(inputText, 'utf16le').toString('utf16le') + : inputText; + batch.push({ + ...input, + chunkIndex: state.chunks++, + startLine: chunk.startLine, + endLine: chunk.endLine, + text: detached, + release, + }); + } + prevTail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + } + state.ended = !failed.has(node.id) && !abort; + node.methodNames = undefined; + node.fieldNames = undefined; + if (abort) break; + } + if (!abort) await flush(); + const finished = [...generated] + .filter(([id, node]) => node.ended && !failed.has(id)) + .map(([nodeId, node]) => ({ nodeId, contentHash: node.contentHash, chunks: node.chunks })); + if (finished.length) + for (const receipt of await store.finishNodes(finished)) + complete.set(receipt.nodeId, receipt); + if (complete.size) await store.checkpointUnit([...complete.values()]); + if (complete.size === unit.nodes.length) { + await journal.acknowledge({ + reusedNodes, + chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), + }); + } else await journal.fail(failureCause); + progress(); + if (abort) throw abort; + } finally { + for (const row of batch) row.release(); + releaseScratch(); + fetched.release(); + } + }; + + const replay = async (intent: EmbeddingUnitIntent): Promise => { + const units = readEmbeddingWorkUnits(directory, manifest, intent.start); + try { + const first = await units.next(); + if ( + first.done || + first.value.next.offset !== intent.next.offset || + first.value.unit.nodes.length !== intent.nodes + ) + throw new Error('Embedding recovery intent does not match the manifest'); + await processUnit(first.value.unit, intent); + } finally { + await units.return(undefined); + } + }; + + if (journal.snapshot().phase === 'embedding') { + const active = journal.snapshot().active; + if (active) await replay(active); + for await (const intent of journal.failed()) await replay(intent); + let cursor = journal.snapshot().cursor; + for await (const { unit, next } of readEmbeddingWorkUnits(directory, manifest, cursor)) { + await processUnit(unit, { + start: cursor, + next, + nodes: unit.nodes.length, + origin: 'manifest', + }); + cursor = next; + } + if (journal.snapshot().failedUnits) + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes }; + await journal.indexing(); + progress(); + } + signal?.throwIfAborted(); + const indexKind = await store.seal(manifest.totalNodes, journal.snapshot().chunks); + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; +} diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 9b70bddaa..236971c97 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -181,7 +181,7 @@ export const contentHashForNode = ( return hash.digest('hex'); }; -const isInputRejection = (error: unknown): boolean => +export const isInputRejection = (error: unknown): boolean => error instanceof HttpEmbeddingError && (error.status === 400 || error.status === 413 || error.status === 422); @@ -193,14 +193,15 @@ type EmbeddedRange = { start: number; end: number } & ( /** Bisect only input rejections, with at most 2n-1 requests per sub-batch. * Authentication, throttling, transport and cancellation retain their existing * bounded retry/failure policy instead of multiplying requests. */ -async function* embedInputRanges( +export async function* embedInputRanges( texts: string[], subBatchSize: number, signal?: AbortSignal, + provider: typeof embedBatch = embedBatch, ): AsyncGenerator { async function* request(start: number, end: number): AsyncGenerator { try { - const embeddings = await embedBatch(texts.slice(start, end), { signal }); + const embeddings = await provider(texts.slice(start, end), { signal }); yield { start, end, embeddings }; } catch (error) { if (signal?.aborted || isCancellationError(error)) throw error; diff --git a/gitnexus/src/core/embeddings/line-index.ts b/gitnexus/src/core/embeddings/line-index.ts index 2dc12de3d..265628e71 100644 --- a/gitnexus/src/core/embeddings/line-index.ts +++ b/gitnexus/src/core/embeddings/line-index.ts @@ -4,11 +4,15 @@ export interface ResolvedLineRange { } export const buildLineIndex = (content: string): Int32Array => { - const offsets: number[] = [0]; - for (let i = 0; i < content.length; i++) { - if (content.charCodeAt(i) === 10) offsets.push(i + 1); - } - return new Int32Array(offsets); + // Allocate the final packed index directly; a temporary number[] costs much + // more than the source for files with millions of short lines. + let count = 1; + for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) count++; + const offsets = new Int32Array(count); + let line = 1; + for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) + offsets[line++] = i + 1; + return offsets; }; const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => { diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index d5ab3f403..32bf70a3c 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -48,6 +48,8 @@ const truncateDescription = (text: string, maxLength: number): string => { * Clean code content for embedding */ const cleanContent = (content: string): string => { + // Keep this hot function small enough for V8 to inline ordinary snippets. + if (content.length >= 16 * 1024) return cleanLargeContent(content); return content .replace(/\r\n/g, '\n') .replace(/\n{3,}/g, '\n\n') @@ -57,6 +59,46 @@ const cleanContent = (content: string): string => { .trim(); }; +const cleanLargeContent = (content: string): string => { + const normalized = content.replace(/\r\n/g, '\n').replace(/\n{3,}/g, '\n\n'); + const output = new TextBlocks(); + for (const line of contentLines(normalized)) output.append(line.trimEnd()); + return output.finish().trim(); +}; + +/** Output-sized storage with a bounded number of temporary line strings. */ +class TextBlocks { + private pending: string[] = []; + private output = ''; + count = 0; + append(line: string): void { + if (this.count++) this.pending.push('\n'); + this.pending.push(line); + if (this.pending.length >= 1024) this.flush(); + } + private flush(): void { + this.output += this.pending.join(''); + this.pending = []; + } + finish(): string { + this.flush(); + return this.output; + } +} + +function* contentLines(content: string): Generator { + let start = 0; + for (;;) { + const end = content.indexOf('\n', start); + if (end < 0) { + yield content.slice(start); + return; + } + yield content.slice(start, end); + start = end + 1; + } +} + /** * Compact location signal for the embedding header: the last 1-2 path segments * (immediate parent dir + basename), never the full deep path. @@ -254,25 +296,35 @@ const DECL_START_RE = * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction). */ export const extractDeclarationOnly = (content: string): string => { - const lines = content.split('\n'); - const declLines: string[] = []; + const declLines = new TextBlocks(); let depth = 0; let started = false; let classDepth = 0; let skipDepth = 0; - for (const [idx, line] of lines.entries()) { + let nextLineStart = 0; + for (const line of contentLines(content)) { + nextLineStart += line.length + 1; const trimmed = line.trim(); if (!started) { if (DECL_START_RE.test(trimmed)) { // Non-brace language check: current line or next 3 lines must have `{` - const nextLines = lines.slice(idx + 1, idx + 4); - if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) { + let lookaheadEnd = nextLineStart; + for (let i = 0; i < 3; i++) { + const end = content.indexOf('\n', lookaheadEnd); + if (end < 0) { + lookaheadEnd = content.length; + break; + } + lookaheadEnd = end + 1; + } + const nextBrace = content.indexOf('{', nextLineStart); + if (!trimmed.includes('{') && !(nextBrace >= 0 && nextBrace < lookaheadEnd)) { return ''; } started = true; - declLines.push(trimmed); + declLines.append(trimmed); for (const ch of trimmed) { if (ch === '{') depth++; else if (ch === '}') depth--; @@ -283,8 +335,12 @@ export const extractDeclarationOnly = (content: string): string => { } // Always update depth (even when skipping) - const opens = (trimmed.match(/{/g) || []).length; - const closes = (trimmed.match(/}/g) || []).length; + let opens = 0; + let closes = 0; + for (let i = 0; i < trimmed.length; i++) { + if (trimmed.charCodeAt(i) === 123) opens++; + else if (trimmed.charCodeAt(i) === 125) closes++; + } const prevDepth = depth; depth += opens - closes; @@ -293,7 +349,7 @@ export const extractDeclarationOnly = (content: string): string => { skipDepth = 0; // Closing brace of class if (depth <= 0) { - declLines.push(trimmed); + declLines.append(trimmed); break; } } @@ -306,7 +362,7 @@ export const extractDeclarationOnly = (content: string): string => { if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) { if (opens === closes && trimmed.endsWith(';')) { // Property with function/object initializer like `config = { timeout: 5000 };` — keep - declLines.push(trimmed); + declLines.append(trimmed); } // else: single-line or multi-line method — skip entirely if (opens !== closes) { @@ -315,12 +371,12 @@ export const extractDeclarationOnly = (content: string): string => { continue; } - declLines.push(trimmed); + declLines.append(trimmed); - if (depth <= 0 && declLines.length > 1) break; + if (depth <= 0 && declLines.count > 1) break; } - return declLines.join('\n').trim(); + return declLines.finish().trim(); }; /** diff --git a/gitnexus/src/core/lbug/embedding-source.ts b/gitnexus/src/core/lbug/embedding-source.ts new file mode 100644 index 000000000..ba26e7ff5 --- /dev/null +++ b/gitnexus/src/core/lbug/embedding-source.ts @@ -0,0 +1,181 @@ +/** Key-only manifest scans and bounded PK fetches against one immutable graph. */ +import lbug from '@ladybugdb/core'; +import { openLbugConnection, type LbugConnectionHandle } from './lbug-config.js'; +import { + EMBEDDABLE_LABELS, + LABELS_WITH_EXPORTED, + LABEL_METHOD, + type EmbeddableNode, +} from '../embeddings/types.js'; +import type { + EmbeddingManifestSource, + EmbeddingWorkKey, +} from '../embeddings/embedding-work-manifest.js'; + +const LABELS = new Set([...EMBEDDABLE_LABELS, 'File']); +// ECMAScript trim's whitespace set, including Unicode separators and BOM. +// RE2's default \\S is ASCII-only and would change the File fallback selection. +const NON_WHITESPACE = + '[^\\x09-\\x0d\\x20\\x{a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]'; + +function checkedLabel(label: string): string { + if (!LABELS.has(label)) throw new Error('Unsupported embedding manifest label'); + return `\`${label}\``; +} + +function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult { + if (!Array.isArray(result)) return result; + if (result.length === 1) return result[0]; + for (const part of result) part.close(); + throw new Error('Unexpected multiple graph query results'); +} + +function size(value: unknown): number { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid graph byte count'); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) throw new Error('Invalid graph byte count'); + return result; +} + +function string(value: unknown): string { + if (typeof value !== 'string') throw new Error('Invalid graph text field'); + return value; +} +function optionalString(value: unknown): string | undefined { + return value == null ? undefined : string(value); +} + +export class EmbeddingGraphSource implements EmbeddingManifestSource { + private active = false; + private closed = false; + private readonly fetchStatements = new Map(); + private constructor(private readonly handle: LbugConnectionHandle) {} + + /** graphPath has already passed the publication layer's ownership validation. */ + static async open( + graphPath: string, + bufferPoolBytes: number, + threads: number, + ): Promise { + return new EmbeddingGraphSource( + await openLbugConnection(lbug, graphPath, { readOnly: true, bufferPoolBytes, threads }), + ); + } + + private acquire(): void { + if (this.closed || this.active) + throw new Error('Embedding graph source is closed or already in use'); + this.active = true; + } + + async *scan(label: string): AsyncGenerator { + const table = checkedLabel(label); + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + const fields = ['id', 'name', 'filePath']; + if (label !== 'File') fields.push('description'); + if (label === LABEL_METHOD) fields.push('returnType'); + // JSON escaping costs at most six bytes per source byte. The fixed + // allowance covers property names, numeric spans, booleans and the label. + const metadataBytes = fields + .map((field) => `octet_length(encode(coalesce(n.${field}, '')))`) + .join(' + '); + const filter = + label === 'File' + ? "WHERE n.filePath IS NOT NULL AND n.filePath <> '' AND n.content <> '[Binary file - content not stored]' AND regexp_matches(coalesce(n.content,''), $nonWhitespace)" + : ''; + const sql = `MATCH (n:${table}) ${filter} RETURN n.id AS id, octet_length(encode(coalesce(n.content,''))) AS contentBytes, 256 + 6 * (${metadataBytes}) AS metadataBytes ORDER BY n.id`; + if (label === 'File') { + const statement = await this.handle.conn.prepare(sql); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + result = single( + await this.handle.conn.execute(statement, { nonWhitespace: NON_WHITESPACE }), + ); + } else result = single(await this.handle.conn.query(sql)); + while (result.hasNext()) { + const row = result.getNextSync(); + yield { + id: string(row.id), + contentBytes: size(row.contentBytes), + metadataBytes: size(row.metadataBytes), + }; + } + } finally { + result?.close(); + this.active = false; + } + } + + async fetch(label: string, ids: readonly string[]): Promise { + const table = checkedLabel(label); + if (!ids.length || ids.length > 64 || new Set(ids).size !== ids.length) + throw new Error('Invalid graph fetch unit'); + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + let statement = this.fetchStatements.get(label); + if (!statement) { + const fields = ['id', 'name', 'filePath', 'content']; + if (label !== 'File') fields.push('startLine', 'endLine', 'description'); + if (LABELS_WITH_EXPORTED.has(label)) fields.push('isExported'); + if (label === LABEL_METHOD) fields.push('parameterCount', 'returnType'); + statement = await this.handle.conn.prepare( + `UNWIND $ids AS id MATCH (n:${table} {id:id}) RETURN ${fields.map((field) => `n.${field} AS ${field}`).join(', ')}`, + ); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + this.fetchStatements.set(label, statement); + } + result = single(await this.handle.conn.execute(statement, { ids: [...ids] })); + const nodes: EmbeddableNode[] = []; + while (result.hasNext()) { + if (nodes.length >= ids.length) throw new Error('Graph fetch exceeded its manifest unit'); + const row = result.getNextSync(); + const content = row.content ?? ''; + if (typeof content !== 'string') throw new Error('Graph content is not text'); + let fileLines = 1; + if (label === 'File') + for (let i = 0; i < content.length; i++) if (content.charCodeAt(i) === 10) fileLines++; + if (row.isExported != null && typeof row.isExported !== 'boolean') + throw new Error('Invalid graph export flag'); + nodes.push({ + id: string(row.id), + name: string(row.name), + label, + filePath: string(row.filePath), + content, + startLine: label === 'File' ? 1 : row.startLine == null ? undefined : size(row.startLine), + endLine: + label === 'File' ? fileLines : row.endLine == null ? undefined : size(row.endLine), + ...(label === 'File' ? {} : { description: optionalString(row.description) }), + ...(LABELS_WITH_EXPORTED.has(label) + ? { isExported: row.isExported == null ? undefined : (row.isExported as boolean) } + : {}), + ...(label === LABEL_METHOD + ? { + parameterCount: row.parameterCount == null ? undefined : size(row.parameterCount), + returnType: optionalString(row.returnType), + } + : {}), + }); + } + return nodes; + } finally { + result?.close(); + this.active = false; + } + } + + async close(): Promise { + if (this.closed) return; + if (this.active) throw new Error('Cannot close an active embedding graph read'); + this.closed = true; + this.fetchStatements.clear(); + try { + await this.handle.conn.close(); + } finally { + await this.handle.db.close(); + } + } +} diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index ce0e8a43e..8b852b534 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -7,7 +7,7 @@ import { ExtensionManager } from './extension-loader.js'; import { assertValidGraphNodeId } from '../graph/node-identity.js'; import { embeddingJobDirectory } from '../../storage/embedding-job.js'; -export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v1'; +export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v2'; const BLOCK_ROWS = 256; const MAX_CHUNKS = 2147483647; @@ -22,10 +22,14 @@ export interface VectorNodeReceipt { contentHash: string; chunks: number; complete: boolean; + sourceStartLine?: number; + sourceEndLine?: number; } export interface VectorNodeInput { nodeId: string; contentHash: string; + sourceStartLine?: number; + sourceEndLine?: number; } export type VectorWriteChunk = StoredVectorChunk & VectorNodeInput; export interface VectorStoreOptions { @@ -105,7 +109,7 @@ export class VectorStore { `CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${options.dimensions}], contentHash STRING, PRIMARY KEY(id))`, ); await store.query( - 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, PRIMARY KEY(nodeId))', + 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', ); await store.query( 'CREATE NODE TABLE EmbeddingArtifact (id STRING, identity STRING, schema STRING, dimensions INT64, complete BOOL, indexKind STRING, PRIMARY KEY(id))', @@ -200,7 +204,7 @@ export class VectorStore { private async receipts(ids: readonly string[]): Promise> { const wanted = new Set(ids); const rows = await this.query( - 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) RETURN n.nodeId AS nodeId, n.contentHash AS contentHash, n.chunks AS chunks, n.complete AS complete', + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) RETURN n.nodeId AS nodeId, n.contentHash AS contentHash, n.chunks AS chunks, n.complete AS complete, n.sourceStartLine AS sourceStartLine, n.sourceEndLine AS sourceEndLine', { ids: [...ids] }, ); const receipts = new Map(); @@ -221,6 +225,8 @@ export class VectorStore { contentHash: row.contentHash, chunks, complete: row.complete, + sourceStartLine: count(row.sourceStartLine), + sourceEndLine: count(row.sourceEndLine), }); } return receipts; @@ -240,7 +246,14 @@ export class VectorStore { throw new Error('Invalid vector node unit'); for (const node of nodes) { assertValidGraphNodeId(node.nodeId, 'Vector store'); - if (!node.contentHash) throw new Error('Missing embedding input hash'); + if (typeof node.contentHash !== 'string' || !node.contentHash) + throw new Error('Missing embedding input hash'); + if ( + !nonnegative(node.sourceStartLine ?? 0) || + !nonnegative(node.sourceEndLine ?? 0) || + (node.sourceEndLine ?? 0) < (node.sourceStartLine ?? 0) + ) + throw new Error('Invalid embedding source lines'); } await this.exclusive(async () => { const previous = await this.receipts(nodes.map((n) => n.nodeId)); @@ -267,16 +280,16 @@ export class VectorStore { await remove(); } await this.transaction(async () => { - for (const { nodeId, contentHash } of nodes) { + for (const { nodeId, contentHash, sourceStartLine = 0, sourceEndLine = 0 } of nodes) { if (previous.has(nodeId)) await this.query( - 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.chunks=0', - { nodeId, contentHash }, + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.chunks=0, n.sourceStartLine=$sourceStartLine, n.sourceEndLine=$sourceEndLine', + { nodeId, contentHash, sourceStartLine, sourceEndLine }, ); else await this.query( - 'CREATE (:EmbeddingNodeReceipt {nodeId:$nodeId, contentHash:$contentHash, chunks:0, complete:false})', - { nodeId, contentHash }, + 'CREATE (:EmbeddingNodeReceipt {nodeId:$nodeId, contentHash:$contentHash, chunks:0, complete:false, sourceStartLine:$sourceStartLine, sourceEndLine:$sourceEndLine})', + { nodeId, contentHash, sourceStartLine, sourceEndLine }, ); } }); @@ -395,6 +408,7 @@ export class VectorStore { throw new Error('Invalid vector completion unit'); return this.exclusive(async () => { const receipts = await this.receipts(nodes.map((n) => n.nodeId)); + const verified: VectorNodeReceipt[] = []; for (const node of nodes) { const receipt = receipts.get(node.nodeId); if ( @@ -405,12 +419,13 @@ export class VectorStore { ) throw new Error('Vector node coverage does not match its generated chunks'); await this.verify(receipt); + verified.push(receipt); } await this.query( 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', { ids: nodes.map((n) => n.nodeId) }, ); - return nodes.map((node) => ({ ...node, complete: true })); + return verified.map((receipt) => ({ ...receipt, complete: true })); }, true); } @@ -424,6 +439,63 @@ export class VectorStore { }); } + /** Same immutable job identity: completed groups need no source/model work. */ + async completedNodes(ids: readonly string[]): Promise> { + if (ids.length > 64 || new Set(ids).size !== ids.length) + throw new Error('Invalid vector receipt unit'); + return this.exclusive(async () => { + const receipts = await this.receipts(ids); + for (const [id, receipt] of receipts) { + if (!receipt.complete || receipt.chunks < 1) receipts.delete(id); + else await this.verify(receipt); + } + return receipts; + }); + } + + /** Bounded copy from a compatible complete artifact; never materializes a node's vectors. */ + async readChunks( + receipt: VectorNodeReceipt, + start: number, + length: number, + ): Promise { + if ( + !nonnegative(start) || + !nonnegative(length) || + !length || + length > BLOCK_ROWS || + start + length > receipt.chunks || + length * (this.options.dimensions * 16 + 128) > this.options.batchBytes + ) + throw new Error('Invalid vector read batch'); + return this.exclusive(async () => { + const rows = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash', + { ids: Array.from({ length }, (_, i) => `${receipt.nodeId}:${start + i}`) }, + ); + if (rows.length !== length) throw new Error('Incomplete vector copy'); + const chunks = rows + .map((row) => { + if ( + row.contentHash !== receipt.contentHash || + !Array.isArray(row.embedding) || + row.embedding.length !== this.options.dimensions + ) + throw new Error('Invalid vector copy'); + return { + chunkIndex: count(row.chunkIndex), + startLine: count(row.startLine), + endLine: count(row.endLine), + embedding: row.embedding as number[], + }; + }) + .sort((a, b) => a.chunkIndex - b.chunkIndex); + if (chunks.some((chunk, i) => chunk.chunkIndex !== start + i)) + throw new Error('Invalid vector copy sequence'); + return chunks; + }); + } + /** Durability must precede the filesystem journal's acknowledgement. */ async checkpointUnit(nodes: readonly VectorNodeReceipt[]): Promise { if ( @@ -444,13 +516,16 @@ export class VectorStore { ) throw new Error('Cannot acknowledge incomplete vector unit'); } + // An already sealed artifact has a durable final checkpoint. Recovery may + // rebuild a damaged filesystem cursor using its verified complete groups. + if (this.sealed) return; try { await this.query('CHECKPOINT'); } catch (error) { this.poisoned = true; throw error; } - }, true); + }, !this.sealed); } /** Runs only once all manifest and failed units have complete coverage. */ @@ -469,6 +544,14 @@ export class VectorStore { count(vectors[0].chunks) !== totalChunks ) throw new Error('Vector artifact coverage is incomplete'); + if (this.sealed) { + const rows = await this.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.indexKind AS kind", + ); + if (rows[0]?.kind !== 'hnsw' && rows[0]?.kind !== 'exact') + throw new Error('Invalid vector index receipt'); + return rows[0].kind; + } const manager = new ExtensionManager(); let kind: 'hnsw' | 'exact' = 'exact'; if ( @@ -498,7 +581,7 @@ export class VectorStore { } this.sealed = true; return kind; - }, true); + }, !this.sealed); } async close(): Promise { diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts index 5538b5c29..2f5c43da7 100644 --- a/gitnexus/src/storage/embedding-job-journal.ts +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -200,6 +200,64 @@ export class EmbeddingJobJournal { return clone(this.state); } + /** Recovery requires an independently verified manifest and vector identity. + * Preserve damaged receipts; the runner reconstructs coverage only from native + * complete groups. A valid receipt for another identity is never discarded. + * This also resumes an interruption between preserving and replacing job.json. + */ + static async recover( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + const canonical = await embeddingJobDirectory(directory); + let value: unknown; + try { + value = await readEmbeddingJobRecord(canonical, 'job.json'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code && code !== 'ENOENT' && code !== 'ERR_ENCODING_INVALID_ENCODED_DATA') throw error; + if ( + !code && + !(error instanceof SyntaxError) && + !/checksum|Malformed|Invalid|UTF-8|receipt exceeds/i.test(String(error)) + ) + throw error; + } + if (value && typeof value === 'object') { + const state = value as Partial; + if ( + (state.version !== undefined && state.version !== 1) || + (state.identity && state.identity !== identity) || + (state.manifestDigest && state.manifestDigest !== manifest.digest) + ) + throw new Error('Embedding job identity mismatch; recovery refused'); + try { + validateState(value, identity, manifest); + return new EmbeddingJobJournal(canonical, value, clone(manifest)); + } catch { + /* Matching identity but damaged accounting requires reconstruction. */ + } + } + const suffix = `.preserved-${randomUUID()}`; + try { + await fs.rename(path.join(canonical, 'job.json'), path.join(canonical, `job.json${suffix}`)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await syncEmbeddingJobDirectory(canonical); + const entries = await fs.opendir(canonical); + for await (const entry of entries) { + if (FAILED.test(entry.name)) + await fs.rename( + path.join(canonical, entry.name), + path.join(canonical, entry.name + suffix), + ); + } + await syncEmbeddingJobDirectory(canonical); + return this.create(canonical, identity, manifest); + } + private async mutate(action: (next: EmbeddingJobState) => Promise): Promise { if (this.busy) throw new Error('Concurrent embedding journal mutation'); if (this.uncertain) diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 6f2f2ba94..58ec5b503 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -5,6 +5,15 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { VectorStore, type VectorStoreOptions } from '../../src/core/lbug/vector-store.js'; import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import { runEmbeddingJob } from '../../src/core/embeddings/embedding-job-runner.js'; +import { + createEmbeddingWorkManifest, + loadEmbeddingWorkManifest, +} from '../../src/core/embeddings/embedding-work-manifest.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { HttpEmbeddingError } from '../../src/core/embeddings/http-client.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; let temporary: TestDBHandle | undefined; let store: VectorStore; @@ -30,6 +39,7 @@ async function setup() { return options; } afterEach(async () => { + vi.restoreAllMocks(); await store?.close(); store = undefined; await temporary?.cleanup(); @@ -37,6 +47,377 @@ afterEach(async () => { vi.unstubAllEnvs(); }); +async function runnerFixture(count: number, content = 'const value = "漢😀";') { + const options = await setup(); + const node = (id: string): EmbeddableNode => ({ + id, + name: id.split(':').at(-1), + label: 'Const', + filePath: 'source.ts', + content, + startLine: 1, + endLine: 1, + }); + const fetch = vi.fn(async (_label: string, ids: readonly string[]) => ids.map(node)); + const source = { + async *scan(label: string) { + if (label !== 'Const') return; + for (let i = 0; i < count; i++) + yield { + id: `Const:source.ts:n${String(i).padStart(5, '0')}`, + contentBytes: Buffer.byteLength(content), + metadataBytes: 1024, + }; + }, + fetch, + }; + const manifest = await createEmbeddingWorkManifest(options.directory, 'source', source); + let journal = await EmbeddingJobJournal.create(options.directory, options.identity, manifest); + const initializeProvider = vi.fn(async () => {}); + const provider = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + const run = (extra = {}) => + runEmbeddingJob({ + directory: options.directory, + manifest, + source, + journal, + store, + queueBytes: 32 * 1024 * 1024, + config: { subBatchSize: 8, maxInputChars: 1024 }, + provider, + initializeProvider, + ...extra, + }); + const reopen = async (recover = false) => { + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + const selection = await loadEmbeddingWorkManifest(options.directory, 'source'); + journal = await (recover ? EmbeddingJobJournal.recover : EmbeddingJobJournal.open).call( + EmbeddingJobJournal, + options.directory, + options.identity, + selection, + ); + }; + return { + options, + source, + manifest, + get journal() { + return journal; + }, + initializeProvider, + provider, + run, + reopen, + }; +} + +describe('bounded durable embedding runner', () => { + it('copies compatible complete vectors into a new generation without re-embedding', async () => { + const fixture = await runnerFixture(3, 'const value = "漢😀";'.repeat(300)); + const first = await fixture.run(); + const previous = store; + const directory = path.join(fixture.options.directory, 'next-generation'); + await fs.mkdir(directory); + const manifest = await createEmbeddingWorkManifest(directory, 'new-source', fixture.source); + const identity = 'b'.repeat(64); + const journal = await EmbeddingJobJournal.create(directory, identity, manifest); + store = await VectorStore.open({ ...fixture.options, directory, identity }); + const shiftedSource = { + ...fixture.source, + fetch: async (label: string, ids: readonly string[]) => + (await fixture.source.fetch(label, ids)).map((node) => ({ + ...node, + startLine: 50, + endLine: 50, + })), + }; + const provider = vi.fn(async () => { + throw new Error('Reuse must not call provider'); + }); + try { + const result = await runEmbeddingJob({ + directory, + manifest, + source: shiftedSource, + journal, + store, + reuse: previous, + provider, + config: { maxInputChars: 1024 }, + queueBytes: 32 * 1024 * 1024, + }); + expect(result.state).toMatchObject({ + reusedNodes: 3, + completedNodes: 0, + chunks: first.state.chunks, + }); + expect(provider).not.toHaveBeenCalled(); + const copied = await store.completedNodes(['Const:source.ts:n00000']); + const receipt = copied.get('Const:source.ts:n00000'); + expect(receipt).toMatchObject({ sourceStartLine: 50, sourceEndLine: 50 }); + expect((await store.readChunks(receipt, 0, 1))[0]).toMatchObject({ + startLine: 50, + endLine: 50, + }); + } finally { + await previous.close(); + } + }); + + it.each(['before', 'after'] as const)( + 'replaces an interrupted chunk group after a %s-write failure', + async (when) => { + const fixture = await runnerFixture(65); + const append = store.appendBatch.bind(store); + let calls = 0; + vi.spyOn(store, 'appendBatch').mockImplementation(async (rows) => { + if (++calls === 9) { + if (when === 'after') await append(rows); + throw new Error('simulated write failure'); + } + await append(rows); + }); + await expect(fixture.run()).rejects.toThrow('write failure'); + expect(fixture.journal.snapshot()).toMatchObject({ + completedNodes: 64, + active: { start: { unit: 1 } }, + }); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).state).toMatchObject({ + completedNodes: 65, + chunks: 65, + failedUnits: 0, + }); + const paid = fixture.provider.mock.calls.flatMap(([texts]) => texts); + expect(paid).toHaveLength(1); + expect(paid[0]).toContain('n00064'); + }, + ); + + it('preserves a corrupt journal and reconstructs complete native coverage without provider calls', async () => { + const fixture = await runnerFixture(65); + await fixture.run(); + const recordPath = path.join(fixture.options.directory, 'job.json'); + const damaged = '{"checksum":"bad","payload":{}}'; + await fs.writeFile(recordPath, damaged); + await fixture.reopen(true); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + const result = await fixture.run(); + expect(result.state).toMatchObject({ + reusedNodes: 65, + chunks: 65, + failedUnits: 0, + phase: 'indexing', + }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(fixture.source.fetch).not.toHaveBeenCalled(); + const saved = (await fs.readdir(fixture.options.directory)).filter((name) => + name.startsWith('job.json.preserved-'), + ); + expect(saved).toHaveLength(1); + expect(await fs.readFile(path.join(fixture.options.directory, saved[0]), 'utf8')).toBe(damaged); + }); + + it('scans only keys and byte bounds, then fetches a bounded PK unit from an immutable graph', async () => { + const options = await setup(); + const graphPath = path.join(options.directory, 'source-graph'); + const handle = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + try { + await query( + 'CREATE NODE TABLE File(id STRING, name STRING, filePath STRING, content STRING, PRIMARY KEY(id))', + ); + await query( + 'CREATE NODE TABLE Function(id STRING, name STRING, filePath STRING, content STRING, description STRING, startLine INT64, endLine INT64, isExported BOOL, PRIMARY KEY(id))', + ); + await query( + "CREATE (:Function {id:'Function:a:漢', name:'漢', filePath:'a.ts', content:'漢😀\ntext', startLine:2, endLine:3, isExported:true})", + ); + await query("CREATE (:File {id:'File:a', name:'a', filePath:'a.md', content:'漢😀\ntext'})"); + await query( + "CREATE (:File {id:'File:blank', name:'blank', filePath:'blank.md', content:'\u00a0\u3000\ufeff'})", + ); + await query( + "CREATE (:File {id:'File:binary', name:'binary', filePath:'binary.dat', content:'[Binary file - content not stored]'})", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle); + } + const source = await EmbeddingGraphSource.open(graphPath, 64 * 1024 * 1024, 1); + try { + const keys = []; + for await (const key of source.scan('Function')) keys.push(key); + expect(keys).toEqual([ + { id: 'Function:a:漢', contentBytes: 12, metadataBytes: expect.any(Number) }, + ]); + const nodes = await source.fetch('Function', [keys[0].id]); + expect(nodes[0]).toMatchObject({ + id: 'Function:a:漢', + content: '漢😀\ntext', + startLine: 2, + endLine: 3, + isExported: true, + }); + expect(keys[0].metadataBytes).toBeGreaterThanOrEqual( + Buffer.byteLength(JSON.stringify({ ...nodes[0], content: undefined })), + ); + const files = []; + for await (const key of source.scan('File')) files.push(key.id); + expect(files).toEqual(['File:a']); + expect((await source.fetch('File', files))[0]).toMatchObject({ startLine: 1, endLine: 2 }); + await expect(source.fetch('Function', Array(65).fill('id'))).rejects.toThrow('unit'); + await expect(source.scan('Unknown').next()).rejects.toThrow('label'); + } finally { + await source.close(); + } + }); + + it('recovers native durability before cursor acknowledgement without repeating paid inputs', async () => { + const fixture = await runnerFixture(130); + const acknowledge = fixture.journal.acknowledge.bind(fixture.journal); + let acks = 0; + vi.spyOn(fixture.journal, 'acknowledge').mockImplementation(async (result) => { + if (++acks === 2) throw new Error('simulated cursor write failure'); + await acknowledge(result); + }); + await expect(fixture.run()).rejects.toThrow('cursor write'); + expect(fixture.journal.snapshot()).toMatchObject({ + completedNodes: 64, + active: { start: { unit: 1 } }, + }); + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + const result = await fixture.run({ + queueBytes: 8 * 1024 * 1024, + config: { subBatchSize: 2, maxInputChars: 1024 }, + }); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(2); + expect(fixture.source.fetch.mock.calls.flatMap(([, ids]) => ids)).toHaveLength(2); + expect(result.state).toMatchObject({ + phase: 'indexing', + completedNodes: 66, + reusedNodes: 64, + failedUnits: 0, + chunks: 130, + }); + // A crash after seal but before publication requires no model/source work. + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + expect((await fixture.run()).indexKind).toBe('exact'); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(fixture.source.fetch).not.toHaveBeenCalled(); + }); + + it('keeps one rejected input retryable while preserving every healthy sibling', async () => { + const fixture = await runnerFixture(12); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00003'))) + throw new HttpEmbeddingError('bad input', { status: 400 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + const first = await fixture.run(); + expect(first.indexKind).toBeUndefined(); + expect(first.state).toMatchObject({ failedUnits: 1, phase: 'embedding', active: null }); + expect( + ( + await store.completedNodes( + Array.from({ length: 12 }, (_, i) => `Const:source.ts:n${String(i).padStart(5, '0')}`), + ) + ).size, + ).toBe(11); + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.provider.mockImplementation(async (texts) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + fixture.source.fetch.mockClear(); + const result = await fixture.run(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(fixture.source.fetch.mock.calls[0][1]).toEqual(['Const:source.ts:n00003']); + expect(result.state).toMatchObject({ + failedUnits: 0, + completedNodes: 1, + reusedNodes: 11, + chunks: 12, + }); + }); + + it('bounds one oversized node by bytes and preserves all generated Unicode input', async () => { + const content = '漢😀abcdef'.repeat(60000); + const fixture = await runnerFixture(1, content); + let chars = 0; + let requests = 0; + // Do not retain the fixture's requests: test the actual production queue. + const provider = async (texts: string[]) => { + requests++; + expect(texts.length).toBeLessThanOrEqual(8); + expect(texts.every((text) => text.length <= 1024)).toBe(true); + expect(texts.every((text) => !/[\uD800-\uDBFF]$/.test(text))).toBe(true); + chars += texts.reduce((n, text) => n + text.length, 0); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }; + const result = await fixture.run({ provider }); + expect(chars).toBeGreaterThan(content.length); + expect(chars).toBeLessThan(content.length + 100); + expect(requests).toBeGreaterThan(50); + expect(result.peakQueueBytes).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(result.state.completedNodes).toBe(1); + }); + + it('rejects impossible admission before intent, provider initialization, or vector mutation', async () => { + const fixture = await runnerFixture(1, 'x'.repeat(100000)); + const begin = vi.spyOn(store, 'beginNodes'); + await expect(fixture.run({ queueBytes: 1024 * 1024 })).rejects.toThrow( + 'larger resource budget', + ); + expect(fixture.initializeProvider).not.toHaveBeenCalled(); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(begin).not.toHaveBeenCalled(); + expect(fixture.journal.snapshot().active).toBeNull(); + }); + + it('does not bisect endpoint outages and stops at the existing consecutive-failure ceiling', async () => { + const fixture = await runnerFixture(100); + fixture.provider.mockRejectedValue(new HttpEmbeddingError('unavailable', { status: 503 })); + await expect(fixture.run()).rejects.toThrow('unavailable'); + expect(fixture.provider).toHaveBeenCalledTimes(5); + expect(fixture.journal.snapshot()).toMatchObject({ + failedUnits: 1, + completedNodes: 0, + active: null, + }); + }); + + it('leaves a durable active unit on cancellation and never writes a late provider response', async () => { + const fixture = await runnerFixture(20); + const controller = new AbortController(); + const append = vi.spyOn(store, 'appendBatch'); + fixture.provider.mockImplementation(async (texts) => { + controller.abort(); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + await expect(fixture.run({ signal: controller.signal })).rejects.toThrow(); + expect(fixture.provider).toHaveBeenCalledTimes(1); + expect(append).not.toHaveBeenCalled(); + expect(fixture.journal.snapshot().active).not.toBeNull(); + }); +}); + describe('separate durable vector artifacts', () => { it('writes a multi-node unit and verifies a large node in bounded batches', async () => { const options = await setup(); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts new file mode 100644 index 000000000..f193a83cc --- /dev/null +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { iterateCharacterChunks } from '../../src/core/embeddings/character-chunk.js'; +import { buildLineIndex } from '../../src/core/embeddings/line-index.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { extractDeclarationOnly } from '../../src/core/embeddings/text-generator.js'; + +const fingerprintNode = (label: string, content: string) => ({ + id: 'Function:src/example.ts:test', + name: 'Example', + filePath: 'src/example.ts', + description: 'Sample description.', + startLine: 1, + endLine: 99, + label, + content, +}); + +describe('bounded embedding text preserves existing vectors', () => { + it.each([ + [ + 'Function', + 'export function 漢() {\r\n return "😀"; \r\n\r\n\r\n}\u3000\n', + '69f18eaa2f90496499546d329389cfbbaf05b5ed', + ], + ['File', 'a \t\r\n\n\n'.repeat(100000), '8750197e52b3b2d3ce89913a1946ff377f3b47e6'], + [ + 'Class', + 'export class Example {\n value: string;\n method() { return "😀"; }\n}\n', + '9559301b3f875312190e654e10b16734631a1dee', + ], + [ + 'TypeAlias', + 'type Huge = ' + '"漢😀" | '.repeat(40000) + 'never;\n', + 'b0b393e258779fafede851f76d33d0831695e6bb', + ], + ])('keeps the pre-change v7 %s fingerprint', (label, content, expected) => { + expect(contentHashForNode(fingerprintNode(label, content))).toBe(expected); + }); + + it('retains every character and exact line spans across Unicode and unusual overlaps', () => { + const content = '\n😀漢\r\n a\n'.repeat(70); + for (const overlap of [0, 1, 7, 8, 20]) { + let covered = 0; + for (const chunk of iterateCharacterChunks(content, 13, 999, 8, overlap)) { + expect(chunk.text).toBe(content.slice(chunk.startOffset, chunk.endOffset)); + expect(chunk.startLine).toBe( + 13 + content.slice(0, chunk.startOffset).split('\n').length - 1, + ); + expect(chunk.endLine).toBe( + 13 + content.slice(0, chunk.endOffset - 1).split('\n').length - 1, + ); + if (overlap >= 0) expect(chunk.startOffset).toBeLessThanOrEqual(covered); + covered = chunk.endOffset; + } + expect(covered).toBe(content.length); + } + }); + + it('produces the first huge-line chunk without retaining the remaining chunks', () => { + const content = '漢😀'.repeat(1000000); + const iterator = iterateCharacterChunks(content, 1, 1, 1200, 120); + expect(iterator.next().value).toMatchObject({ + text: content.slice(0, 1200), + startOffset: 0, + endOffset: 1200, + startLine: 1, + endLine: 1, + }); + iterator.return(undefined); + expect(iterator.next().done).toBe(true); + }); + + it('packs line offsets including the final empty line', () => { + expect(buildLineIndex('a\r\n😀\n\n')).toEqual(new Int32Array([0, 3, 6, 7])); + const index = buildLineIndex('a\n'.repeat(1000000)); + expect(index.length).toBe(1000001); + expect(index.at(-1)).toBe(2000000); + }); + + it('keeps exactly three lines of declaration brace lookahead', () => { + expect(extractDeclarationOnly('class Example\n\n\n{\n field: string;\n}')).toBe( + 'class Example', + ); + expect(extractDeclarationOnly('class Example\n\n\n\n{\n field: string;\n}')).toBe(''); + }); +}); From 167074ffeb3eb12ae8338647bf94e60e3450c743 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 13:44:32 +0000 Subject: [PATCH 06/67] feat(storage): add generation publication Publish durable graph generations independently of vector jobs and gate readers on matching identities. Retain the legacy graph inode under a private name so older binaries cannot serve a stale canonical graph. Coordinator integration follows separately. Refs #3598 --- .../hooks/registry-query.cjs | 32 +- .../hooks/registry-query.cjs | 32 +- .../hooks/registry-query.cjs | 32 +- gitnexus/hooks/claude/registry-query.cjs | 32 +- gitnexus/src/core/analysis-publication.ts | 303 ++++++++++++++++ .../src/core/embeddings/embedding-pipeline.ts | 148 ++++---- .../embeddings/vector-generation-search.ts | 130 +++++++ .../src/core/embeddings/vector-publication.ts | 87 +++++ gitnexus/src/core/lbug/pool-adapter.ts | 132 ++++++- gitnexus/src/core/lbug/vector-store.ts | 36 ++ gitnexus/src/mcp/local/local-backend.ts | 227 +++++++----- gitnexus/src/storage/repo-manager.ts | 7 +- gitnexus/src/storage/repo-meta.ts | 27 +- gitnexus/src/storage/shared-store.ts | 32 +- gitnexus/src/storage/vector-generation.ts | 99 ++++++ .../embedding-vector-store.test.ts | 336 ++++++++++++++++++ .../test/unit/analysis-publication.test.ts | 184 ++++++++++ gitnexus/test/unit/hooks-shared-store.test.ts | 56 +++ gitnexus/test/unit/lbug-vector-pool.test.ts | 103 ++++++ .../local-backend-vector-generation.test.ts | 79 ++++ gitnexus/test/unit/vector-generation.test.ts | 99 ++++++ 21 files changed, 1994 insertions(+), 219 deletions(-) create mode 100644 gitnexus/src/core/analysis-publication.ts create mode 100644 gitnexus/src/core/embeddings/vector-generation-search.ts create mode 100644 gitnexus/src/core/embeddings/vector-publication.ts create mode 100644 gitnexus/src/storage/vector-generation.ts create mode 100644 gitnexus/test/unit/analysis-publication.test.ts create mode 100644 gitnexus/test/unit/lbug-vector-pool.test.ts create mode 100644 gitnexus/test/unit/local-backend-vector-generation.test.ts create mode 100644 gitnexus/test/unit/vector-generation.test.ts diff --git a/gitnexus-claude-plugin/hooks/registry-query.cjs b/gitnexus-claude-plugin/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-claude-plugin/hooks/registry-query.cjs +++ b/gitnexus-claude-plugin/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus-cursor-integration/hooks/registry-query.cjs b/gitnexus-cursor-integration/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-cursor-integration/hooks/registry-query.cjs +++ b/gitnexus-cursor-integration/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus-factory-plugin/hooks/registry-query.cjs b/gitnexus-factory-plugin/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-factory-plugin/hooks/registry-query.cjs +++ b/gitnexus-factory-plugin/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus/hooks/claude/registry-query.cjs b/gitnexus/hooks/claude/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus/hooks/claude/registry-query.cjs +++ b/gitnexus/hooks/claude/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts new file mode 100644 index 000000000..52f14ab3b --- /dev/null +++ b/gitnexus/src/core/analysis-publication.ts @@ -0,0 +1,303 @@ +/** Durable graph publication. Native verification/close and metadata ownership + * are supplied by the coordinator, which holds the index-directory write lock. */ +import fs from 'node:fs/promises'; +import { constants } from 'node:fs'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { INDEX_METADATA_FILE } from '../storage/storage-constants.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; + +const RECEIPT = 'analysis-publication.json'; +const GENERATION = /^[a-f0-9]{32}$/; +const DIGEST = /^[a-f0-9]{64}$/; +const STAGING = /^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; +const MAX_RECEIPT_BYTES = 4096; +const MAX_METADATA_BYTES = 256 * 1024 * 1024; +const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); + +export interface PreparedAnalysisPublication { + version: 1; + id: string; + stagingFile: string; + graphFile: string; + metadataFile: string; + metadataBytes: number; + metadataDigest: string; + /** A hard-link destination for the previous legacy graph; never copied. */ + legacyGraphFile?: string; +} + +export interface AnalysisPublicationActions { + /** Open read-only and verify graph/schema/FTS against the prepared metadata. + * Must close the verification handle before returning (including on Windows). + * Unknown counts/catalog state must reject, never certify an empty graph. + */ + verify: (graphPath: string, metadata: RepoMeta) => Promise; + /** The normal metadata writer, including its compatibility mirror. */ + saveMetadata: (metadata: RepoMeta) => Promise; +} + +function validateReceipt(value: unknown): asserts value is PreparedAnalysisPublication { + const r = value as PreparedAnalysisPublication; + if ( + !r || + r.version !== 1 || + !GENERATION.test(r.id) || + !STAGING.test(r.stagingFile) || + r.graphFile !== `lbug.generation.${r.id}` || + r.metadataFile !== `analysis-meta-${r.id}.json` || + (r.legacyGraphFile !== undefined && + (!/^lbug\.generation\.[a-f0-9]{32}$/.test(r.legacyGraphFile) || + r.legacyGraphFile === r.graphFile)) || + !Number.isSafeInteger(r.metadataBytes) || + r.metadataBytes < 1 || + r.metadataBytes > MAX_METADATA_BYTES || + !DIGEST.test(r.metadataDigest) + ) + throw new Error('Invalid prepared graph publication; artifacts retained'); +} + +async function regularFile(file: string): Promise { + try { + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Publication artifact must be an owned regular file'); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } +} + +async function boundedRead(file: string, maximum: number): Promise { + if (!(await regularFile(file))) + throw Object.assign(new Error('Publication artifact missing'), { code: 'ENOENT' }); + const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > maximum) + throw new Error('Publication artifact exceeds its receipt bound'); + const buffer = Buffer.alloc(stat.size + 1); + let size = 0; + while (size < buffer.length) { + const result = await handle.read(buffer, size, buffer.length - size, size); + if (!result.bytesRead) break; + size += result.bytesRead; + } + if (size !== stat.size) throw new Error('Publication artifact changed while reading'); + return buffer.subarray(0, size); + } finally { + await handle.close(); + } +} + +async function writeDurable(directory: string, name: string, bytes: Buffer): Promise { + const temporary = path.join(directory, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(directory, name)); + await syncEmbeddingJobDirectory(directory); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +/** Call after graph/FTS verification and a successful native checkpoint, before + * native close. A close crash leaves the owned build and its publication intent. + * No model or vector work may start until publishPreparedAnalysis completes. + */ +export async function prepareAnalysisPublication( + directory: string, + buildPath: string, + metadata: RepoMeta, + coveredContent: string, +): Promise { + const root = await embeddingJobDirectory(directory); + if ( + !DIGEST.test(coveredContent) || + !metadata.repoPath || + !metadata.runnerIdentity || + metadata.incrementalInProgress + ) + throw new Error('Graph publication requires verified source and analyzer metadata'); + if (path.dirname(path.resolve(buildPath)) !== root || !STAGING.test(path.basename(buildPath))) + throw new Error('Graph build is outside its owned staging directory'); + if (!(await regularFile(buildPath))) throw new Error('Graph build is missing'); + if (await regularFile(path.join(root, RECEIPT))) + throw new Error('Recover the pending graph publication before preparing another'); + const id = randomUUID().replaceAll('-', ''); + const graphFile = `lbug.generation.${id}`; + const metadataFile = `analysis-meta-${id}.json`; + const preparedMeta: RepoMeta = { + ...metadata, + graphPath: graphFile, + graphGeneration: { version: 1, id, coveredContent }, + }; + const bytes = Buffer.from(JSON.stringify(preparedMeta)); + if (bytes.length > MAX_METADATA_BYTES) + throw new Error('Graph metadata exceeds the supported publication size'); + const receipt: PreparedAnalysisPublication = { + version: 1, + id, + stagingFile: path.basename(buildPath), + graphFile, + metadataFile, + metadataBytes: bytes.length, + metadataDigest: hash(bytes), + ...((await regularFile(path.join(root, 'lbug'))) + ? { legacyGraphFile: `lbug.generation.${randomUUID().replaceAll('-', '')}` } + : {}), + }; + validateReceipt(receipt); + await writeDurable(root, metadataFile, bytes); + const payload = JSON.stringify(receipt); + await writeDurable( + root, + RECEIPT, + Buffer.from(JSON.stringify({ checksum: hash(payload), payload: receipt })), + ); + return receipt; +} + +export async function readPreparedAnalysis( + directory: string, +): Promise { + const root = await embeddingJobDirectory(directory); + let bytes: Buffer; + try { + bytes = await boundedRead(path.join(root, RECEIPT), MAX_RECEIPT_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + const envelope: unknown = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + if ( + !envelope || + typeof envelope !== 'object' || + !('payload' in envelope) || + !('checksum' in envelope) || + hash(JSON.stringify(envelope.payload)) !== envelope.checksum + ) + throw new Error('Prepared publication checksum mismatch; artifacts retained'); + validateReceipt(envelope.payload); + return envelope.payload; +} + +export async function syncPublishedMetadata(root: string, metadata: RepoMeta): Promise { + const metadataPath = path.join(root, INDEX_METADATA_FILE); + const saved = await boundedRead(metadataPath, MAX_METADATA_BYTES); + if ( + hash(JSON.stringify(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(saved)))) !== + hash(JSON.stringify(metadata)) + ) + throw new Error( + 'Published graph metadata does not match the prepared generation; artifacts retained', + ); + // Windows FlushFileBuffers requires a writable descriptor. The coordinator + // owns the write lock; this opens the file without changing its contents. + const handle = await fs.open(metadataPath, constants.O_RDWR | (constants.O_NOFOLLOW ?? 0)); + try { + await handle.sync(); + } finally { + await handle.close(); + } + await syncEmbeddingJobDirectory(root); +} + +/** Older binaries ignore private generation pointers. Remove the obsolete + * canonical name before advancing metadata, retaining its inode for readers + * and rollback. A hard-link failure leaves the old index readable and aborts; + * relocating vectors must never trigger a full graph copy. + */ +async function retireLegacyGraph( + root: string, + receipt: PreparedAnalysisPublication, + metadata: RepoMeta, + actions: AnalysisPublicationActions, +): Promise { + const legacy = path.join(root, 'lbug'); + if (!receipt.legacyGraphFile) { + if (await regularFile(legacy)) + throw new Error('Unexpected legacy graph appeared during publication'); + return; + } + const retired = path.join(root, receipt.legacyGraphFile); + if (!(await regularFile(legacy))) { + if (!(await regularFile(retired))) throw new Error('Legacy graph retirement artifact missing'); + return; + } + if (await regularFile(retired)) { + const [before, after] = await Promise.all([fs.stat(legacy), fs.stat(retired)]); + if (!before.ino || before.ino !== after.ino || before.dev !== after.dev) + throw new Error('Conflicting legacy graph retirement artifact'); + } else { + await fs.link(legacy, retired); + await syncEmbeddingJobDirectory(root); + } + const previous = JSON.parse( + (await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES)).toString('utf8'), + ) as RepoMeta; + if (previous.repoPath !== metadata.repoPath) + throw new Error('Legacy graph belongs to another repository'); + if (!previous.graphPath) { + const redirected = { ...previous, graphPath: receipt.legacyGraphFile }; + await actions.saveMetadata(redirected); + await syncPublishedMetadata(root, redirected); + } + await fs.unlink(legacy); + await syncEmbeddingJobDirectory(root); +} + +/** Also recovers crashes after rename or metadata publication. Every attempt + * re-verifies the owned native artifact; a generation identifier is not proof. + * Existing graph files remain untouched so readers can finish their snapshots. + */ +export async function publishPreparedAnalysis( + directory: string, + actions: AnalysisPublicationActions, +): Promise { + const root = await embeddingJobDirectory(directory); + const receipt = await readPreparedAnalysis(root); + if (!receipt) return undefined; + const bytes = await boundedRead(path.join(root, receipt.metadataFile), receipt.metadataBytes); + if (bytes.length !== receipt.metadataBytes || hash(bytes) !== receipt.metadataDigest) + throw new Error('Prepared graph metadata checksum mismatch; artifacts retained'); + const metadata = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as RepoMeta; + if ( + metadata.graphPath !== receipt.graphFile || + metadata.graphGeneration?.version !== 1 || + metadata.graphGeneration.id !== receipt.id || + !DIGEST.test(metadata.graphGeneration.coveredContent) + ) + throw new Error('Prepared graph generation mismatch'); + const staged = path.join(root, receipt.stagingFile); + const published = path.join(root, receipt.graphFile); + const stagedExists = await regularFile(staged); + const publishedExists = await regularFile(published); + if (stagedExists === publishedExists) + throw new Error('Prepared publication has missing or conflicting graph artifacts'); + const source = stagedExists ? staged : published; + await actions.verify(source, metadata); + if (stagedExists) { + await fs.rename(staged, published); + await syncEmbeddingJobDirectory(root); + } + await retireLegacyGraph(root, receipt, metadata, actions); + await actions.saveMetadata(metadata); + // The canonical pointer must be durable before callers start paid work. + await syncPublishedMetadata(root, metadata); + await fs.unlink(path.join(root, RECEIPT)); + // Persist removal of the recovery pointer before removing what it names. + // A power loss must not resurrect a receipt whose metadata is already gone. + await syncEmbeddingJobDirectory(root); + await fs.unlink(path.join(root, receipt.metadataFile)); + await syncEmbeddingJobDirectory(root); + return metadata; +} diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 236971c97..7ec5a6f95 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -65,6 +65,7 @@ import { needsEmbeddingSource, type EmbeddingSourceContext, } from './source-reader.js'; +import type { VectorSearchSnapshot } from './vector-generation-search.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -1281,34 +1282,52 @@ export const semanticSearch = async ( query: string, k: number = 10, maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), + snapshot?: VectorSearchSnapshot, ): Promise => { - // determinism: probe — existence only. Only `exists.length` is read; which - // row LIMIT 1 returns cannot change whether the table is empty. - const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`); - if (!exists.length) { - return []; - } - - if (!isEmbedderReady()) { - throw new Error('Embedding model not initialized. Run embedding pipeline first.'); - } - - const queryEmbedding = await embedText(query); - const queryVec = embeddingToArray(queryEmbedding); - const queryVecStr = `[${queryVec.join(',')}]`; - let bestChunks = new Map< string, { distance: number; chunkIndex: number; startLine: number; endLine: number } >(); - // Query/read path: NEVER spawn a network INSTALL on a user query. If the - // VECTOR extension was not pre-installed, fall back to exact scan rather than - // blocking the query on a download (offline-first; see extension-loader.ts - // "load-only" — used by all serve/MCP query paths). - if (await loadVectorExtension(undefined, { policy: 'load-only' })) { - try { - bestChunks = await collectBestChunks(k, async (fetchLimit) => { - const vectorQuery = ` + let generationSelected = false; + if (snapshot) { + const { searchVectorGeneration } = await import('./vector-generation-search.js'); + const chunks = await searchVectorGeneration( + snapshot, + async () => { + if (!isEmbedderReady()) + throw new Error('Embedding model not initialized. Run embedding pipeline first.'); + return embeddingToArray(await embedText(query)); + }, + k, + maxDistance, + ); + generationSelected = chunks !== null; + if (chunks) bestChunks = new Map(chunks.map(({ nodeId, ...chunk }) => [nodeId, chunk])); + } + if (!generationSelected) { + // determinism: probe — existence only. Only `exists.length` is read; which + // row LIMIT 1 returns cannot change whether the table is empty. + const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`); + if (!exists.length) { + return []; + } + + if (!isEmbedderReady()) { + throw new Error('Embedding model not initialized. Run embedding pipeline first.'); + } + + const queryEmbedding = await embedText(query); + const queryVec = embeddingToArray(queryEmbedding); + const queryVecStr = `[${queryVec.join(',')}]`; + + // Query/read path: NEVER spawn a network INSTALL on a user query. If the + // VECTOR extension was not pre-installed, fall back to exact scan rather than + // blocking the query on a download (offline-first; see extension-loader.ts + // "load-only" — used by all serve/MCP query paths). + if (await loadVectorExtension(undefined, { policy: 'load-only' })) { + try { + bestChunks = await collectBestChunks(k, async (fetchLimit) => { + const vectorQuery = ` CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit}) YIELD node AS emb, distance @@ -1319,54 +1338,54 @@ export const semanticSearch = async ( ORDER BY distance `; - const embResults = await executeQuery(vectorQuery); - return embResults.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - distance: row.distance ?? row[4], - })); - }); - } catch { - bestChunks = new Map(); + const embResults = await executeQuery(vectorQuery); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); + } catch { + bestChunks = new Map(); + } } - } - if (bestChunks.size === 0) { - // NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher - // never writes a fabricated 0, whereas here `?? 0` is the right answer — - // an unknown count simply skips the exact scan. - const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); - const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0); - const exactLimit = getExactScanLimit(); - if (embeddingCount > 0 && embeddingCount <= exactLimit) { - const rows = await executeQuery(` + if (bestChunks.size === 0) { + // NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher + // never writes a fabricated 0, whereas here `?? 0` is the right answer — + // an unknown count simply skips the exact scan. + const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); + const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0); + const exactLimit = getExactScanLimit(); + if (embeddingCount > 0 && embeddingCount <= exactLimit) { + const rows = await executeQuery(` MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding `); - const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - embedding: row.embedding ?? row[4] ?? [], - })); - bestChunks = new Map( - rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [ - row.nodeId, - { - distance: row.distance, - chunkIndex: row.chunkIndex, - startLine: row.startLine, - endLine: row.endLine, - }, - ]), - ); + const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + embedding: row.embedding ?? row[4] ?? [], + })); + bestChunks = new Map( + rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [ + row.nodeId, + { + distance: row.distance, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + }, + ]), + ); + } } } - if (bestChunks.size === 0) { return []; } @@ -1432,8 +1451,9 @@ export const semanticSearchWithContext = async ( query: string, k: number = 5, _hops: number = 1, + snapshot?: VectorSearchSnapshot, ): Promise => { - const results = await semanticSearch(executeQuery, query, k); + const results = await semanticSearch(executeQuery, query, k, undefined, snapshot); return results.map((r) => ({ matchId: r.nodeId, diff --git a/gitnexus/src/core/embeddings/vector-generation-search.ts b/gitnexus/src/core/embeddings/vector-generation-search.ts new file mode 100644 index 000000000..92323616d --- /dev/null +++ b/gitnexus/src/core/embeddings/vector-generation-search.ts @@ -0,0 +1,130 @@ +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { createHash } from 'node:crypto'; +import type { EmbeddingIdentity } from './embedding-identity.js'; +import { + assertOwnedVectorGeneration, + selectVectorGeneration, +} from '../../storage/vector-generation.js'; +import { + initLbug, + pinRepo, + executeParameterized, + ensureVectorExtension, +} from '../lbug/pool-adapter.js'; +import { VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; +import { getExactScanLimit } from '../platform/capabilities.js'; +import { collectBestChunks } from './types.js'; +import { rankExactEmbeddingRows, type ExactSearchChunk } from './exact-search.js'; +import { assertQueryIdentity, rethrowSymbolIdentityError } from '../lbug/query-result-integrity.js'; + +export interface VectorSearchSnapshot { + metadataDirectory: string; + metadata: RepoMeta | null; + graphPath: string; +} + +const configuredIdentity = async (): Promise => { + const { resolveEmbeddingIdentity } = await import('./embedding-identity.js'); + return resolveEmbeddingIdentity(); +}; + +/** null means a legacy monolithic reader is required. An empty array is a + * deliberate unavailable lane and must not fall through to legacy vectors. */ +export async function searchVectorGeneration( + snapshot: VectorSearchSnapshot, + embed: () => Promise, + limit: number, + maxDistance: number, + resolveIdentity: () => Promise = configuredIdentity, +): Promise { + const exactLimit = getExactScanLimit(); + const selected = selectVectorGeneration( + snapshot.metadataDirectory, + snapshot.metadata, + snapshot.graphPath, + exactLimit, + ); + if (selected.kind === 'legacy') return null; + if (selected.kind !== 'ready') return []; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000 || !Number.isFinite(maxDistance)) + throw new Error('Invalid vector search bounds'); + const dbPath = selected.path; + const release = pinRepo(dbPath); + try { + await assertOwnedVectorGeneration(snapshot.metadataDirectory, selected.generation.identity); + await initLbug(dbPath, dbPath, { + kind: 'vector', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 2, + }); + const execute = (query: string, parameters: Record = {}) => + executeParameterized(dbPath, query, parameters); + const artifact = await execute( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.identity AS identity, a.schema AS schema, a.dimensions AS dimensions, a.complete AS complete, a.indexKind AS kind", + ); + const proof = artifact[0]; + if ( + artifact.length !== 1 || + proof.identity !== selected.generation.identity || + proof.schema !== VECTOR_STORE_SCHEMA || + Number(proof.dimensions) !== selected.generation.dimensions || + proof.complete !== true || + proof.kind !== selected.generation.indexKind + ) + throw new Error('Published vector identity is unverified'); + const annReady = + selected.generation.indexKind === 'hnsw' && + (await ensureVectorExtension(dbPath).catch(() => false)); + if (!annReady && selected.generation.chunks > exactLimit) return []; + const identity = await resolveIdentity(); + const vectorIdentity = createHash('sha256') + .update(JSON.stringify([identity.provider, identity.model, identity.dimensions])) + .digest('hex'); + if (vectorIdentity !== selected.generation.vectorIdentity) return []; + // Neither a metadata pointer nor a timestamp alone authorizes a model call. + const query = await embed(); + if ( + query.length !== selected.generation.dimensions || + query.some((value) => !Number.isFinite(value)) + ) + throw new Error('Query embedding does not match the published vector dimensions'); + if (annReady) { + try { + const best = await collectBestChunks(limit, async (fetchLimit) => { + const rows = await execute( + `CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', $vector, $limit) + YIELD node AS emb, distance WITH emb, distance WHERE distance < $distance + RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, emb.startLine AS startLine, emb.endLine AS endLine, distance ORDER BY distance`, + { vector: [...query], limit: fetchLimit, distance: maxDistance }, + ); + return rows.map((row) => { + assertQueryIdentity(row, 'nodeId', 0); + return row as ExactSearchChunk; + }); + }); + return [...best].map(([nodeId, chunk]) => ({ nodeId, ...chunk })); + } catch (error) { + rethrowSymbolIdentityError(error); + if (selected.generation.chunks > exactLimit) throw error; + } + } + if (selected.generation.chunks > exactLimit) return []; + // LIMIT is a defensive bound against edited metadata, not pagination. + const rows = + await execute(`MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, + e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding LIMIT ${exactLimit + 1}`); + if (rows.length !== selected.generation.chunks || rows.length > exactLimit) + throw new Error('Published vector coverage does not match its receipt'); + for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); + const seen = new Set(); + return rankExactEmbeddingRows(rows, query, rows.length, maxDistance) + .filter((row) => { + if (seen.has(row.nodeId)) return false; + seen.add(row.nodeId); + return true; + }) + .slice(0, limit); + } finally { + release(); + } +} diff --git a/gitnexus/src/core/embeddings/vector-publication.ts b/gitnexus/src/core/embeddings/vector-publication.ts new file mode 100644 index 000000000..c0a514162 --- /dev/null +++ b/gitnexus/src/core/embeddings/vector-publication.ts @@ -0,0 +1,87 @@ +import path from 'node:path'; +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { + assertOwnedVectorGeneration, + selectVectorGeneration, + type PublishedVectorGeneration, +} from '../../storage/vector-generation.js'; +import { VectorStore } from '../lbug/vector-store.js'; +import { syncPublishedMetadata } from '../analysis-publication.js'; +import { getExactScanLimit } from '../platform/capabilities.js'; + +export interface VectorPublicationActions { + loadMetadata: () => Promise; + saveMetadata: (metadata: RepoMeta) => Promise; +} + +/** Caller holds the checkout's index lock and has closed the vector writer. + * A failed verify/close/save leaves the job in indexing, with no paid replay. + * null means ANN remains pending beyond the exact-scan policy limit. + */ +export async function publishVectorGeneration( + metadataDirectory: string, + generation: PublishedVectorGeneration, + resources: { bufferPoolBytes: number; threads: number; batchBytes: number }, + actions: VectorPublicationActions, +): Promise { + const exactScanLimit = getExactScanLimit(); + if (generation.indexKind === 'exact' && generation.chunks > exactScanLimit) return null; + const previous = await actions.loadMetadata(); + if ( + !previous?.graphGeneration || + previous.incrementalInProgress || + previous.graphGeneration.id !== generation.graphGeneration || + previous.embeddingJob?.identity !== generation.identity || + previous.embeddingJob?.graphGeneration !== generation.graphGeneration + ) + throw new Error('Vector publication does not match the active graph job'); + const metadata: RepoMeta = { + ...previous, + vectorGeneration: generation, + embeddingJob: undefined, + embeddingCheckpoint: undefined, + embeddingDims: generation.dimensions, + stats: { ...previous.stats, embeddings: generation.chunks }, + capabilities: { + graph: previous.capabilities?.graph ?? { provider: 'ladybugdb', status: 'available' }, + fts: previous.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' }, + vectorSearch: { + provider: generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: generation.chunks + ? generation.indexKind === 'hnsw' + ? 'vector-index' + : 'exact-scan' + : 'unavailable', + exactScanLimit, + }, + }, + }; + const selection = selectVectorGeneration( + metadataDirectory, + metadata, + path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'), + exactScanLimit, + ); + if ( + selection.kind !== 'ready' && + !(selection.kind === 'unavailable' && selection.reason === 'embeddings-empty') + ) + throw new Error('Invalid vector publication receipt'); + const file = await assertOwnedVectorGeneration(metadataDirectory, generation.identity); + const store = await VectorStore.open({ + directory: path.dirname(file), + identity: generation.identity, + dimensions: generation.dimensions, + mode: 'read', + ...resources, + }); + try { + if ((await store.seal(generation.nodes, generation.chunks)) !== generation.indexKind) + throw new Error('Published vector index does not match the native artifact'); + } finally { + await store.close(); + } + await actions.saveMetadata(metadata); + await syncPublishedMetadata(metadataDirectory, metadata); + return metadata; +} diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index 458c29918..23d845bbe 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -68,6 +68,14 @@ interface PoolEntry { dbIdentity: DbIdentity | null; /** Set to true when the pool entry is closed — checkin will close orphaned connections */ closed: boolean; + artifactOptions?: VectorPoolOptions; +} + +/** Separate vector files never inherit the graph's default native pool. */ +export interface VectorPoolOptions { + kind: 'vector'; + bufferPoolBytes: number; + threads: number; } /** Filesystem identity used to detect an index rebuilt/mutated under a live @@ -157,6 +165,7 @@ interface SharedDB { dbIdentity?: DbIdentity | null; /** When true, closeOne skips db.close() — the Database is owned externally. */ external?: boolean; + artifactOptions?: VectorPoolOptions; } const dbCache = new Map(); @@ -348,12 +357,17 @@ export const getMaxResidentRepos = (): number => MAX_POOL_SIZE; * entry is pinned, no eviction occurs and the pool transiently exceeds * MAX_POOL_SIZE (see the pinnedRepos docstring). */ -async function evictLRU(): Promise { - if (pool.size < MAX_POOL_SIZE) return; +async function evictLRU(kind?: 'vector'): Promise { + // A vector artifact must not halve the existing graph-cache capacity. Each + // lane has its own bounded allowance; vector handles reserve only 64 MiB. + let residents = 0; + for (const entry of pool.values()) if (entry.artifactOptions?.kind === kind) residents++; + if (residents < MAX_POOL_SIZE) return; let oldestId: string | null = null; let oldestTime = Infinity; for (const [id, entry] of pool) { + if (entry.artifactOptions?.kind !== kind) continue; if (pinnedRepos.has(id)) continue; if (entry.checkedOut === 0 && entry.lastUsed < oldestTime) { oldestTime = entry.lastUsed; @@ -783,10 +797,24 @@ function ladybugUnavailableError(repoId: string, err: Error | undefined): Error * bookkeeping on "did the pool actually roll over" (LocalBackend) use the * return value; callers that only need the pool ready can ignore it. */ -export const initLbug = async (repoId: string, dbPath: string): Promise => { +export const initLbug = async ( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise => { + if ( + artifactOptions && + (artifactOptions.kind !== 'vector' || + !Number.isSafeInteger(artifactOptions.bufferPoolBytes) || + artifactOptions.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(artifactOptions.threads) || + artifactOptions.threads < 1) + ) + throw new Error('Invalid vector reader resource reservation'); + const reservation = artifactOptions ? Object.freeze({ ...artifactOptions }) : undefined; let lastError: Error | undefined; for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { - const result = await withPoolLock(() => initLbugInner(repoId, dbPath)); + const result = await withPoolLock(() => initLbugInner(repoId, dbPath, reservation)); if (result.status === 'done') return result.reopened; lastError = result.error; if (attempt === LOCK_RETRY_ATTEMPTS) break; @@ -795,7 +823,11 @@ export const initLbug = async (repoId: string, dbPath: string): Promise throw ladybugUnavailableError(repoId, lastError); }; -const initLbugInner = async (repoId: string, dbPath: string): Promise => { +const initLbugInner = async ( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise => { const existing = pool.get(repoId); if (existing) { existing.lastUsed = Date.now(); @@ -804,7 +836,12 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise 1) { + if (existing.dbPath === dbPath && shared && shared.db === existing.db && shared.refCount > 1) { return { status: 'done', reopened: false }; } // Awaited: see the rationale on the evictLRU call site in doInitLbug below. await closeOne(repoId); // idle & changed → evict, then fall through to reopen the new file } - return doInitLbug(repoId, dbPath); + return doInitLbug(repoId, dbPath, artifactOptions); }; +function sameArtifactOptions(a?: VectorPoolOptions, b?: VectorPoolOptions): boolean { + return ( + a?.kind === b?.kind && a?.bufferPoolBytes === b?.bufferPoolBytes && a?.threads === b?.threads + ); +} + +/** Published vectors are immutable: readers never quarantine, replay, or repair + * their writer's sidecars. An unfinished artifact must stay unavailable. */ +async function openVectorReadOnlyDatabase( + dbPath: string, + options: VectorPoolOptions, +): Promise { + const stat = await fs.lstat(dbPath); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector artifact must be a regular file'); + for (const suffix of ['.wal', '.shadow']) { + try { + const sidecar = await fs.lstat(dbPath + suffix); + if (sidecar.size || sidecar.isSymbolicLink()) + throw new Error('Published vector artifact has unfinished native writes'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + silenceStdout(); + let db: lbug.Database | undefined; + try { + db = createLbugDatabase(lbug, toNativeSafePath(dbPath), { + readOnly: true, + bufferPoolBytes: options.bufferPoolBytes, + }); + await db.init(); + return db; + } catch (error) { + if (db) await db.close().catch(() => {}); + throw error; + } finally { + restoreStdout(); + } +} + /** * Internal init — creates DB, pre-warms connections, loads FTS, then registers pool. * Pool entry is registered LAST so concurrent executeQuery calls see either * "not initialized" (and throw) or a fully ready pool — never a half-built one. */ -async function doInitLbug(repoId: string, dbPath: string): Promise { +async function doInitLbug( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise { // Check if database exists try { await fs.stat(dbPath); @@ -854,11 +936,13 @@ async function doInitLbug(repoId: string, dbPath: string): Promise connection.close())); + shared.refCount--; + if (shared.refCount === 0) { + dbCache.delete(dbPath); + if (!shared.external) await db.close().catch(() => {}); + } + throw error; } finally { preWarmActive = false; } @@ -955,7 +1058,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise this.query(sql), 'vector', 'vector publication', { + policy: 'load-only', + })) + ) + throw new Error('Cannot verify the published vector index'); + await this.verifyVectorIndex(); + } return rows[0].kind; } const manager = new ExtensionManager(); @@ -569,6 +580,17 @@ export class VectorStore { if (/already exists/i.test(String(error))) kind = 'hnsw'; } } + if (kind === 'hnsw') { + try { + await this.verifyVectorIndex(); + } catch { + kind = 'exact'; + } + } + // Large exact scans are deliberately unavailable to readers. Keep this + // complete work unpublished and writable so a later run can retry ANN + // without a single new provider request. + if (kind === 'exact' && totalChunks > getExactScanLimit()) return kind; await this.query( "MATCH (a:EmbeddingArtifact {id:'artifact'}) SET a.complete=true, a.indexKind=$kind", { kind }, @@ -584,6 +606,20 @@ export class VectorStore { }, !this.sealed); } + private async verifyVectorIndex(): Promise { + const sample = await this.query( + 'MATCH (e:CodeEmbedding) RETURN e.embedding AS embedding LIMIT 1', + ); + if (sample.length !== 1 || !Array.isArray(sample[0].embedding)) + throw new Error('Vector index has no probe row'); + const hits = await this.query( + "CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', $vector, 1) YIELD node AS emb, distance RETURN emb.id AS id", + { vector: sample[0].embedding as number[] }, + ); + if (hits.length !== 1 || typeof hits[0].id !== 'string') + throw new Error('Vector index cannot retrieve a stored row'); + } + async close(): Promise { if (this.closed) return; if (this.busy) throw new Error('Cannot close an active vector operation'); diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 54c7e0e1b..2cdeca6ed 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -116,6 +116,8 @@ import { embeddingDimsMismatch, } from '../../core/lbug/schema.js'; import { getExactScanLimit } from '../../core/platform/capabilities.js'; +import { metadataMatchesGraph, selectVectorGeneration } from '../../storage/vector-generation.js'; +import { pinRepo } from '../../core/lbug/pool-adapter.js'; import { PhaseTimer } from '../../core/search/phase-timer.js'; import { ftsDegradedWarning, ftsQueryFailedWarning } from '../../core/search/fts-indexes.js'; import { getFtsDisabledReason, type FtsDisabledReason } from '../../core/search/fts-policy.js'; @@ -1850,6 +1852,17 @@ export class LocalBackend { if (!pending) { pending = loadOptionalMeta(path.dirname(repo.lbugPath)).then((meta) => { if (!meta) return null; + if ( + (meta.graphPath !== undefined || meta.graphGeneration) && + !metadataMatchesGraph(path.dirname(repo.lbugPath), meta, repo.lbugPath) + ) { + // A newer immutable path was published after this request resolved + // its handle. Keep the old reader's metadata, never mix generations. + const previous = this.lastObservedPoolState.get(repo.lbugPath); + if (previous?.metadata) return previous.metadata; + repo.generationUnknown = true; + return null; + } // Readers never use ingestion manifests. Do not keep their per-file // hashes/cache keys or pending embedding IDs in an opened snapshot. return { @@ -3562,6 +3575,7 @@ export class LocalBackend { chain_depth?: number; }, ): Promise { + const release = pinRepo(repo.lbugPath); try { const result = await this._queryImpl(repo, params); if (!result.error) assertResponseIdentities(result); @@ -3571,6 +3585,8 @@ export class LocalBackend { return { error: err.message, recoverySuggestion: SYMBOL_IDENTITY_RECOVERY_SUGGESTION }; } throw err; + } finally { + release(); } } @@ -4575,55 +4591,86 @@ export class LocalBackend { // call's width, and the catch below must only clear an entry it did not set. let embeddedDims: number | undefined; try { - // Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off) - // determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only - // the count is read, to decide whether to load the embedding model at all. - const tableCheck = await executeQuery( + if (repo.generationUnknown) return []; + const metadata = await this.readToolMetadata(repo); + if (repo.generationUnknown) return []; + const selection = selectVectorGeneration( + path.dirname(repo.lbugPath), + metadata, repo.lbugPath, - `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`, + getExactScanLimit(), ); - if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) { + if (selection.kind === 'unavailable') { this.lastQueryEmbeddingDims.delete(repo.lbugPath); - this.logKeywordOnlyEmbeddings(); return []; } - - const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); - const queryVec = await embedQuery(query); - const dims = getEmbeddingDims(); - // #2798: the width this query vector really was produced at — the same - // value the CAST below binds against the index's `FLOAT[N]` column, and - // therefore the only honest comparand for query()'s drift warning. - embeddedDims = dims; - this.lastQueryEmbeddingDims.set(repo.lbugPath, dims); - const queryVecStr = `[${queryVec.join(',')}]`; - const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE); - let vectorReady = false; - try { - vectorReady = await ensureVectorExtension(repo.lbugPath); - } catch (err) { - rethrowSymbolIdentityError(err); - if (!this.warnedVectorLoadFailed) { - this.warnedVectorLoadFailed = true; - logger.warn( - { err }, - 'GitNexus [query:vector]: vector extension load failed; using exact scan fallback', - ); - } - } - let bestChunks = new Map< string, { distance: number; chunkIndex: number; startLine: number; endLine: number } >(); - // Try the vector lane only after its lazy load succeeds. An unavailable - // extension is already reported by ExtensionManager; an index/query - // failure below gets its own once-per-backend diagnostic before the exact - // scan fallback. - if (vectorReady) { + if (selection.kind === 'ready') { + const { searchVectorGeneration } = + await import('../../core/embeddings/vector-generation-search.js'); + const chunks = await searchVectorGeneration( + { metadataDirectory: path.dirname(repo.lbugPath), metadata, graphPath: repo.lbugPath }, + async () => { + const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); + const queryVec = await embedQuery(query); + embeddedDims = getEmbeddingDims(); + this.lastQueryEmbeddingDims.set(repo.lbugPath, embeddedDims); + return queryVec; + }, + limit, + getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE), + ); + bestChunks = new Map((chunks ?? []).map(({ nodeId, ...chunk }) => [nodeId, chunk])); + if (embeddedDims === undefined) this.lastQueryEmbeddingDims.delete(repo.lbugPath); + } else { + // Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off) + // determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only + // the count is read, to decide whether to load the embedding model at all. + const tableCheck = await executeQuery( + repo.lbugPath, + `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`, + ); + if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) { + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + this.logKeywordOnlyEmbeddings(); + return []; + } + + const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); + const queryVec = await embedQuery(query); + const dims = getEmbeddingDims(); + // #2798: the width this query vector really was produced at — the same + // value the CAST below binds against the index's `FLOAT[N]` column, and + // therefore the only honest comparand for query()'s drift warning. + embeddedDims = dims; + this.lastQueryEmbeddingDims.set(repo.lbugPath, dims); + const queryVecStr = `[${queryVec.join(',')}]`; + const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE); + let vectorReady = false; try { - bestChunks = await collectBestChunks(limit, async (fetchLimit) => { - const vectorQuery = ` + vectorReady = await ensureVectorExtension(repo.lbugPath); + } catch (err) { + rethrowSymbolIdentityError(err); + if (!this.warnedVectorLoadFailed) { + this.warnedVectorLoadFailed = true; + logger.warn( + { err }, + 'GitNexus [query:vector]: vector extension load failed; using exact scan fallback', + ); + } + } + + // Try the vector lane only after its lazy load succeeds. An unavailable + // extension is already reported by ExtensionManager; an index/query + // failure below gets its own once-per-backend diagnostic before the exact + // scan fallback. + if (vectorReady) { + try { + bestChunks = await collectBestChunks(limit, async (fetchLimit) => { + const vectorQuery = ` CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit}) YIELD node AS emb, distance @@ -4634,66 +4681,66 @@ export class LocalBackend { ORDER BY distance `; - const embResults = await executeQuery(repo.lbugPath, vectorQuery); - for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0); - return embResults.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - distance: row.distance ?? row[4], - })); - }); - } catch (err) { - rethrowSymbolIdentityError(err); - bestChunks = new Map(); - if (!this.warnedVectorQueryFailed) { - // Rare diagnostic: surface why semantic search fell back to the - // exact scan. Emitted once per `LocalBackend` instance lifetime to - // avoid noisy stderr on hot semantic-search paths (DoD §2.8). - this.warnedVectorQueryFailed = true; - logger.warn( - { err }, - 'GitNexus [query:vector]: vector index query failed; using exact scan fallback', - ); + const embResults = await executeQuery(repo.lbugPath, vectorQuery); + for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); + } catch (err) { + rethrowSymbolIdentityError(err); + bestChunks = new Map(); + if (!this.warnedVectorQueryFailed) { + // Rare diagnostic: surface why semantic search fell back to the + // exact scan. Emitted once per `LocalBackend` instance lifetime to + // avoid noisy stderr on hot semantic-search paths (DoD §2.8). + this.warnedVectorQueryFailed = true; + logger.warn( + { err }, + 'GitNexus [query:vector]: vector index query failed; using exact scan fallback', + ); + } } } - } - if (bestChunks.size === 0) { - const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0); - const exactLimit = getExactScanLimit(); - if (embeddingCount > exactLimit) return []; + if (bestChunks.size === 0) { + const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0); + const exactLimit = getExactScanLimit(); + if (embeddingCount > exactLimit) return []; - const rows = await executeQuery( - repo.lbugPath, - ` + const rows = await executeQuery( + repo.lbugPath, + ` MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding `, - ); - for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); - const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - embedding: row.embedding ?? row[4] ?? [], - })); - bestChunks = new Map( - rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [ - row.nodeId, - { - distance: row.distance, - chunkIndex: row.chunkIndex, - startLine: row.startLine, - endLine: row.endLine, - }, - ]), - ); + ); + for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); + const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + embedding: row.embedding ?? row[4] ?? [], + })); + bestChunks = new Map( + rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [ + row.nodeId, + { + distance: row.distance, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + }, + ]), + ); + } } - if (bestChunks.size === 0) return []; const results: any[] = []; diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index f2f0086e5..d338b71ae 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -50,7 +50,6 @@ import { type AnalyzerRunnerIdentity, type RepoMeta, } from './repo-meta.js'; -import { LBUG_DIRECTORY } from './storage-constants.js'; import { resolveGraphPath } from './shared-store.js'; import { defaultStoragePath, @@ -243,9 +242,9 @@ export const getStoragePaths = ( const baseDir = branch ? path.join(storagePath, BRANCHES_DIR, branchSlug(branch)) : storagePath; return { storagePath, - // Branch slots are always private; a flat shared-store slot may read a - // commit graph (#3352). - lbugPath: branch ? path.join(baseDir, LBUG_DIRECTORY) : resolveGraphPath(storagePath), + // Private generations also work in branch slots. Only a flat shared-store + // slot may resolve to an immutable commit graph (#3352). + lbugPath: resolveGraphPath(baseDir), metaPath: path.join(baseDir, INDEX_METADATA_FILE), // Branch-specific metadata file }; }; diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 40bc80494..594599cf6 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -114,11 +114,32 @@ export interface RepoMeta { /** Complete index directory selected for this successful analysis. */ storagePath?: string; /** - * Shared-store checkouts only (#3352): the graph this slot reads when it is - * not `/lbug` — a commit graph under the same store. Resolved - * and validated by `resolveGraphPath`. + * The graph this slot reads: an owned private generation basename, or an + * absolute shared commit graph under the same store (#3352). Resolved and + * validated by `resolveGraphPath`. Absent means legacy `/lbug`. */ graphPath?: string; + /** A verified immutable graph, published independently of optional vectors. */ + graphGeneration?: { + version: 1; + id: string; + /** Digest of covered content, including uncommitted source changes. */ + coveredContent: string; + }; + /** Checkout-owned durable work. Progress lives in its bounded journal. */ + embeddingJob?: { version: 1; identity: string; graphGeneration: string }; + /** Published only after complete coverage, native durability and index validation. */ + vectorGeneration?: { + version: 1; + identity: string; + graphGeneration: string; + vectorIdentity: string; + inputIdentity: string; + dimensions: number; + nodes: number; + chunks: number; + indexKind: 'hnsw' | 'exact'; + }; lastCommit: string; indexedAt: string; /** Missing on legacy metadata means the upstream-compatible `full` profile. */ diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index a8da36260..3654e8615 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -224,10 +224,10 @@ export const storeRootOfCheckoutSlot = (storagePath: string): string | null => { }; /** - * The graph a flat storage slot reads. + * The graph a storage slot reads. * - * Non-shared storage is always `/lbug`, with no I/O. A shared - * checkout slot may record `graphPath` in its metadata, naming a commit graph + * A private generation is an owned basename in the slot. A shared checkout + * slot may record `graphPath` in its metadata, naming a commit graph * in the same store; any other recorded value (outside the store, a sibling's * private slot, unreadable metadata) falls back to the slot's own graph so a * hand-edited file cannot redirect reads. @@ -244,13 +244,16 @@ const readRecordedGraphPath = (slot: string): unknown => { const metaPath = path.resolve(root, INDEX_METADATA_FILE); const rel = path.relative(root, metaPath); if (rel.startsWith('..') || path.isAbsolute(rel)) return undefined; + const current = fs.statSync(metaPath); + const cached = recordedGraphCache.get(metaPath); + if (cached?.key === `${current.ino}:${current.size}:${current.mtimeMs}:${current.ctimeMs}`) + return cached.recorded; // One open, then stat and read through the same descriptor, so the cache // key always describes the bytes that were parsed. const fd = fs.openSync(metaPath, 'r'); try { const stat = fs.fstatSync(fd); - const key = `${stat.ino}:${stat.size}:${stat.mtimeMs}`; - const cached = recordedGraphCache.get(metaPath); + const key = `${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; if (cached?.key === key) return cached.recorded; const recorded = (JSON.parse(fs.readFileSync(fd, 'utf-8')) as { graphPath?: unknown }) .graphPath; @@ -263,15 +266,28 @@ const readRecordedGraphPath = (slot: string): unknown => { export const resolveGraphPath = (storagePath: string): string => { const own = path.join(storagePath, LBUG_DIRECTORY); - const root = storeRootOfCheckoutSlot(storagePath); - if (!root) return own; let recorded: unknown; try { recorded = readRecordedGraphPath(storagePath); } catch { return own; } - if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; + if (typeof recorded !== 'string') return own; + if (/^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + // Never follow an edited pointer to a symlink or another slot. A missing + // committed generation must fail at open, rather than serve an older lbug. + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Published graph generation must be an owned regular file'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + return graph; + } + const root = storeRootOfCheckoutSlot(storagePath); + if (!root || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); const commitDir = path.dirname(graph); // Only a published `-` dir, never `.publish-*` staging. diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts new file mode 100644 index 000000000..15f29de71 --- /dev/null +++ b/gitnexus/src/storage/vector-generation.ts @@ -0,0 +1,99 @@ +/** Filesystem-only vector routing. No DB or embedding runtime is loaded here. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { RepoMeta } from './repo-meta.js'; + +export type PublishedVectorGeneration = NonNullable; +export type VectorReadSelection = + | { kind: 'legacy' } + | { kind: 'unavailable'; reason: string } + | { kind: 'ready'; path: string; generation: PublishedVectorGeneration }; +const DIGEST = /^[a-f0-9]{64}$/; +const GENERATION = /^[a-f0-9]{32}$/; +const nonnegative = (n: unknown): n is number => + typeof n === 'number' && Number.isSafeInteger(n) && n >= 0; + +export function embeddingGenerationDirectory(metadataDirectory: string, identity: string): string { + if (!DIGEST.test(identity)) throw new Error('Invalid embedding generation identity'); + return path.join(metadataDirectory, 'embeddings', identity); +} + +/** Metadata must describe the exact graph handle used for result hydration. */ +export function metadataMatchesGraph( + metadataDirectory: string, + metadata: RepoMeta, + graphPath: string, +): boolean { + const recorded = metadata.graphPath; + if (recorded !== undefined && typeof recorded !== 'string') return false; + return path.resolve(metadataDirectory, recorded ?? 'lbug') === path.resolve(graphPath); +} + +/** Run before model initialization, vector COUNTs, or opening an artifact. */ +export function selectVectorGeneration( + metadataDirectory: string, + metadata: RepoMeta | null | undefined, + graphPath: string, + exactScanLimit: number, +): VectorReadSelection { + if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob) + return { kind: 'legacy' }; + const graph = metadata?.graphGeneration; + const vector = metadata?.vectorGeneration; + if ( + !graph || + graph.version !== 1 || + !GENERATION.test(graph.id) || + !DIGEST.test(graph.coveredContent) || + !metadataMatchesGraph(metadataDirectory, metadata, graphPath) + ) + return { kind: 'unavailable', reason: 'graph-generation-unverified' }; + if (!vector) + return { + kind: 'unavailable', + reason: metadata.embeddingJob ? 'embedding-pending' : 'embeddings-unavailable', + }; + if (metadata.embeddingJob && metadata.embeddingJob.identity !== vector.identity) + return { kind: 'unavailable', reason: 'embedding-pending' }; + if ( + vector.version !== 1 || + vector.graphGeneration !== graph.id || + !DIGEST.test(vector.identity) || + !DIGEST.test(vector.vectorIdentity) || + !DIGEST.test(vector.inputIdentity) || + !nonnegative(vector.dimensions) || + !vector.dimensions || + !nonnegative(vector.nodes) || + !nonnegative(vector.chunks) || + vector.chunks < vector.nodes || + (vector.indexKind !== 'hnsw' && vector.indexKind !== 'exact') + ) + return { kind: 'unavailable', reason: 'vector-generation-mismatch' }; + if (!vector.chunks) return { kind: 'unavailable', reason: 'embeddings-empty' }; + if (vector.indexKind === 'exact' && vector.chunks > exactScanLimit) + return { kind: 'unavailable', reason: 'vector-index-pending' }; + return { + kind: 'ready', + path: path.join(embeddingGenerationDirectory(metadataDirectory, vector.identity), 'vectors'), + generation: vector, + }; +} + +/** Refuse ancestor symlinks as well as a symlinked database. The coordinator + * holds the slot lock while creating jobs; published directories are immutable. */ +export async function assertOwnedVectorGeneration( + metadataDirectory: string, + identity: string, +): Promise { + const directory = embeddingGenerationDirectory(metadataDirectory, identity); + for (const target of [metadataDirectory, path.dirname(directory), directory]) { + const stat = await fs.lstat(target); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Vector generation must be an owned directory'); + } + const file = path.join(directory, 'vectors'); + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector generation must be an owned regular file'); + return file; +} diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 58ec5b503..881ca92e4 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -14,6 +14,17 @@ import { import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; import { HttpEmbeddingError } from '../../src/core/embeddings/http-client.js'; import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { createHash, randomUUID } from 'node:crypto'; +import { searchVectorGeneration } from '../../src/core/embeddings/vector-generation-search.js'; +import * as readerPool from '../../src/core/lbug/pool-adapter.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import { publishVectorGeneration } from '../../src/core/embeddings/vector-publication.js'; +import { + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { ExtensionManager } from '../../src/core/lbug/extension-loader.js'; let temporary: TestDBHandle | undefined; let store: VectorStore; @@ -565,3 +576,328 @@ describe('separate durable vector artifacts', () => { expect((await fs.stat(path.join(options.directory, 'vectors'))).isFile()).toBe(true); }); }); + +describe('published vector readers', () => { + it('publishes a native FTS graph before opening the independent vector writer', async (context) => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(64 * 1024 * 1024)); + const root = temporary.dbPath; + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + const graph = await openLbugConnection(lbug, stage, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const query = async (sql: string) => { + const result = await graph.conn.query(sql); + const rows = []; + for (const part of Array.isArray(result) ? result : [result]) { + try { + rows.push(...(await part.getAll())); + } finally { + await part.close(); + } + } + return rows; + }; + let prepared; + try { + const ready = await new ExtensionManager().ensure(query, 'fts', 'publication fixture', { + policy: 'load-only', + }); + if (!ready) { + context.skip('Optional native FTS extension is unavailable'); + return; + } + await query('CREATE NODE TABLE Function (id STRING, name STRING, PRIMARY KEY(id))'); + await query("CREATE (:Function {id:'Function:source.ts:alpha', name:'alpha'})"); + await query("CALL CREATE_FTS_INDEX('Function', 'function_fts', ['name'])"); + await query('CHECKPOINT'); + prepared = await prepareAnalysisPublication( + root, + stage, + { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + stats: { nodes: 1 }, + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }, + 'c'.repeat(64), + ); + } finally { + await closeLbugConnection(graph); + } + const metadata = await publishPreparedAnalysis(root, { + verify: async (file) => { + await readerPool.initLbug('verify-publication', file); + try { + expect( + await readerPool.executeQuery( + 'verify-publication', + "CALL QUERY_FTS_INDEX('Function', 'function_fts', 'alpha') YIELD node, score RETURN node.name AS name", + ), + ).toEqual([{ name: 'alpha' }]); + } finally { + await readerPool.closeLbug('verify-publication'); + } + }, + saveMetadata: (meta) => saveMeta(root, meta), + }); + expect(metadata?.graphPath).toBe(prepared.graphFile); + const publishedPath = path.join(root, prepared.graphFile); + const directory = path.join(root, 'embeddings', 'd'.repeat(64)); + await fs.mkdir(directory, { recursive: true }); + try { + await readerPool.initLbug('warm-fts', publishedPath); + store = await VectorStore.open({ + directory, + identity: 'd'.repeat(64), + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0)]); + await readerPool.initLbug('cold-fts', publishedPath); + for (const key of ['warm-fts', 'cold-fts']) { + expect( + await readerPool.executeQuery( + key, + "CALL QUERY_FTS_INDEX('Function', 'function_fts', 'alpha') YIELD node, score RETURN node.name AS name", + ), + ).toEqual([{ name: 'alpha' }]); + } + } finally { + await readerPool.closeLbug(); + } + }); + + it('uses the native ANN lane when exact scan cannot cover the artifact', async (context) => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '1'); + const root = temporary.dbPath; + const probe = await openLbugConnection(lbug, path.join(root, 'extension-probe'), { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + const loaded = await probe.conn.query('LOAD vector'); + for (const result of Array.isArray(loaded) ? loaded : [loaded]) { + await result.getAll(); + await result.close(); + } + } catch { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } finally { + await closeLbugConnection(probe); + } + const generation = 'a'.repeat(32); + const identity = 'b'.repeat(64); + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + const options: VectorStoreOptions = { + directory, + identity, + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }; + store = await VectorStore.open(options); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + await store.finishNode('Function:source.ts:alpha', 'hash', 2); + const kind = await store.seal(1, 2); + expect(kind).toBe('hnsw'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.seal(1, 2)).toBe('hnsw'); + await store.close(); + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: `lbug.generation.${generation}`, + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: createHash('sha256') + .update(JSON.stringify(['local', 'fixture', 4])) + .digest('hex'), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind: kind, + }, + }; + try { + const found = await searchVectorGeneration( + { metadataDirectory: root, metadata, graphPath: path.join(root, metadata.graphPath!) }, + async () => [0.1, 0.2, 0.3, 0.4], + 5, + 1, + async () => ({ provider: 'local', model: 'fixture', dimensions: 4 }), + ); + expect(found).toEqual([ + expect.objectContaining({ nodeId: 'Function:source.ts:alpha', startLine: 1, endLine: 2 }), + ]); + } finally { + await readerPool.closeLbug(); + } + }); + + it('serves warm and cold graph readers while vectors are unpublished, then searches the closed artifact concurrently', async () => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(64 * 1024 * 1024)); + const root = temporary.dbPath; + const generation = 'a'.repeat(32); + const identity = 'b'.repeat(64); + const graphPath = path.join(root, `lbug.generation.${generation}`); + const graph = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + for (const query of [ + 'CREATE NODE TABLE Function (id STRING, name STRING, PRIMARY KEY(id))', + "CREATE (:Function {id:'Function:source.ts:alpha', name:'alpha'})", + 'CHECKPOINT', + ]) { + const result = await graph.conn.query(query); + for (const part of Array.isArray(result) ? result : [result]) { + await part.getAll(); + await part.close(); + } + } + } finally { + await closeLbugConnection(graph); + } + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + store = await VectorStore.open({ + directory, + identity, + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }); + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: path.basename(graphPath), + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + embeddingJob: { version: 1, identity, graphGeneration: generation }, + }; + const snapshot = { metadataDirectory: root, metadata, graphPath }; + await saveMeta(root, metadata); + const embed = vi.fn(async () => [0.1, 0.2, 0.3, 0.4]); + const vectorIdentity = { provider: 'local', model: 'fixture', dimensions: 4 }; + const resolveIdentity = async () => vectorIdentity; + try { + await readerPool.initLbug(graphPath, graphPath); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + expect(await searchVectorGeneration(snapshot, embed, 5, 1, resolveIdentity)).toEqual([]); + expect(embed).not.toHaveBeenCalled(); + await readerPool.initLbug('cold-reader', graphPath); + for (const key of [graphPath, 'cold-reader']) { + expect( + await readerPool.executeQuery(key, 'MATCH (n:Function) RETURN n.name AS name'), + ).toEqual([{ name: 'alpha' }]); + } + const receipt = await store.finishNode('Function:source.ts:alpha', 'hash', 2); + await store.checkpointUnit([receipt]); + const indexKind = await store.seal(1, 2); + await store.close(); + const publishedGeneration: NonNullable = { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: createHash('sha256') + .update(JSON.stringify(['local', 'fixture', 4])) + .digest('hex'), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind, + }; + const resources = { bufferPoolBytes: 64 * 1024 * 1024, threads: 1, batchBytes: 64 * 1024 }; + const actions = { + loadMetadata: () => loadMeta(root), + saveMetadata: (meta: RepoMeta) => saveMeta(root, meta), + }; + await expect( + publishVectorGeneration(root, { ...publishedGeneration, nodes: 2 }, resources, actions), + ).rejects.toThrow('incomplete'); + expect((await loadMeta(root))?.vectorGeneration).toBeUndefined(); + const published = await publishVectorGeneration( + root, + publishedGeneration, + resources, + actions, + ); + expect(published?.embeddingJob).toBeUndefined(); + expect(published?.stats?.embeddings).toBe(2); + snapshot.metadata = published!; + const results = await Promise.all( + Array.from({ length: 8 }, () => + searchVectorGeneration(snapshot, embed, 5, 1, resolveIdentity), + ), + ); + for (const result of results) + expect(result).toEqual([ + expect.objectContaining({ + nodeId: 'Function:source.ts:alpha', + chunkIndex: 0, + startLine: 1, + endLine: 2, + }), + ]); + expect(embed).toHaveBeenCalledTimes(8); + embed.mockClear(); + expect( + await searchVectorGeneration(snapshot, embed, 5, 1, async () => ({ + ...vectorIdentity, + model: 'different', + })), + ).toEqual([]); + expect(embed).not.toHaveBeenCalled(); + } finally { + await readerPool.closeLbug(); + } + }); + + it('keeps complete vectors retryable when exact coverage exceeds the reader limit', async () => { + const options = await setup(); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '1'); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + await store.finishNode('Function:source.ts:alpha', 'hash', 2); + expect(await store.seal(1, 2)).toBe('exact'); + await store.close(); + await expect(VectorStore.open({ ...options, mode: 'read' })).rejects.toThrow('not complete'); + store = await VectorStore.open({ ...options, mode: 'resume' }); + expect((await store.completedNodes(['Function:source.ts:alpha'])).size).toBe(1); + // Restoring a viable exact policy exercises index-only completion; no + // replacement or provider call is needed to publish the existing rows. + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '2'); + expect(await store.seal(1, 2)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect((await store.completedNodes(['Function:source.ts:alpha'])).size).toBe(1); + }); +}); diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts new file mode 100644 index 000000000..bf474455f --- /dev/null +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -0,0 +1,184 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { getStoragePaths, saveMeta } from '../../src/storage/repo-manager.js'; +import { resolveGraphPath } from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const directories: string[] = []; +const content = 'a'.repeat(64); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abcdef', + indexedAt: '2026-11-03T00:00:00Z', + // Publication preserves the independently verified runner receipt verbatim. + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], +}; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publication-'))); + directories.push(root); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + await fs.writeFile(path.join(root, 'lbug'), 'previous graph'); + await saveMeta(root, metadata); + const verify = vi.fn(async (file: string) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }); + const actions = { verify, saveMetadata: (meta: RepoMeta) => saveMeta(root, meta) }; + return { root, stage, actions }; +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('durable graph publication', () => { + it('publishes a verified generation and keeps the previous reader artifact', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + expect(resolveGraphPath(root)).toBe(path.join(root, 'lbug')); + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration).toEqual({ + version: 1, + id: receipt.id, + coveredContent: content, + }); + expect(actions.verify).toHaveBeenCalledOnce(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + if (!receipt.legacyGraphFile) throw new Error('Expected a retained legacy graph'); + expect(await fs.readFile(path.join(root, receipt.legacyGraphFile), 'utf8')).toBe( + 'previous graph', + ); + await expect(fs.stat(path.join(root, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + expect(await publishPreparedAnalysis(root, actions)).toBeUndefined(); + expect(await fs.readdir(root)).not.toContain(receipt.metadataFile); + }); + + it.each(['before-rename', 'after-rename', 'after-metadata'] as const)( + 'recovers a crash %s and re-verifies the native artifact', + async (point) => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + if (point === 'before-rename') { + await expect( + publishPreparedAnalysis(root, { + ...actions, + verify: async () => { + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + } else { + await expect( + publishPreparedAnalysis(root, { + ...actions, + saveMetadata: async (meta) => { + if (point === 'after-metadata') await actions.saveMetadata(meta); + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + } + expect(await readPreparedAnalysis(root)).toEqual(receipt); + actions.verify.mockClear(); + await publishPreparedAnalysis(root, actions); + expect(actions.verify).toHaveBeenCalledOnce(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + }, + ); + + it('retains recovery evidence when the metadata writer fails to publish the exact receipt', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + await expect( + publishPreparedAnalysis(root, { ...actions, saveMetadata: async () => {} }), + ).rejects.toThrow('does not match'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + await publishPreparedAnalysis(root, actions); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + }); + + it('keeps a completed publication usable when redundant metadata cleanup fails', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const unlink = fs.unlink.bind(fs); + vi.spyOn(fs, 'unlink').mockImplementation(async (file) => { + if (String(file) === path.join(root, receipt.metadataFile)) throw new Error('cleanup failed'); + await unlink(file); + }); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('cleanup failed'); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + expect(await publishPreparedAnalysis(root, actions)).toBeUndefined(); + }); + + it.each(['receipt', 'metadata', 'graph'] as const)( + 'rejects corrupted %s without publishing', + async (artifact) => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const file = + artifact === 'receipt' + ? 'analysis-publication.json' + : artifact === 'metadata' + ? receipt.metadataFile + : receipt.stagingFile; + await fs.writeFile(path.join(root, file), 'corrupt'); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow(); + expect(resolveGraphPath(root)).toBe(path.join(root, 'lbug')); + expect(await fs.readdir(root)).toContain('analysis-publication.json'); + }, + ); + + it('refuses missing or conflicting graph artifacts', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + await fs.writeFile(path.join(root, receipt.graphFile), 'conflict'); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('conflicting'); + await fs.unlink(path.join(root, receipt.graphFile)); + await fs.unlink(stage); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('missing'); + expect(actions.verify).not.toHaveBeenCalled(); + }); + + it('rejects outside paths, symlinks and a second uncommitted publication', async () => { + const { root, stage } = await fixture(); + await expect( + prepareAnalysisPublication( + root, + path.join(root, '..', path.basename(stage)), + metadata, + content, + ), + ).rejects.toThrow('outside'); + const link = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.symlink(stage, link); + await expect(prepareAnalysisPublication(root, link, metadata, content)).rejects.toThrow( + 'regular file', + ); + await prepareAnalysisPublication(root, stage, metadata, content); + await expect(prepareAnalysisPublication(root, stage, metadata, content)).rejects.toThrow( + 'Recover', + ); + }); + + it('resolves private generations in branch slots and never falls back from a missing published file', async () => { + const { root } = await fixture(); + const slot = path.dirname(getStoragePaths('/repo', 'feature', root).metaPath); + const graphPath = `lbug.generation.${'b'.repeat(32)}`; + await saveMeta(slot, { ...metadata, graphPath }); + expect(getStoragePaths('/repo', 'feature', root).lbugPath).toBe(path.join(slot, graphPath)); + await fs.symlink(path.join(root, 'lbug'), path.join(slot, graphPath)); + expect(() => getStoragePaths('/repo', 'feature', root)).toThrow('regular file'); + }); +}); diff --git a/gitnexus/test/unit/hooks-shared-store.test.ts b/gitnexus/test/unit/hooks-shared-store.test.ts index 074767d3d..e9c65faef 100644 --- a/gitnexus/test/unit/hooks-shared-store.test.ts +++ b/gitnexus/test/unit/hooks-shared-store.test.ts @@ -5,6 +5,7 @@ import path from 'path'; import { createRequire } from 'module'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { slotNameForCanonicalPath } from '../../src/storage/storage-slot.js'; +import { branchSlug } from '../../src/storage/branch-index.js'; /** * #3352 — the Claude hook resolves a shared-store checkout to the commit @@ -102,6 +103,61 @@ describe('registry-query shared store graph (#3352)', () => { for (const copy of copies) expect(copy).toBe(primary); }); + it.each(HOOK_COPIES)('resolves a private generation in a registered branch slot (%s)', (file) => { + const branch = 'feature/private'; + execFileSync( + 'git', + [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.invalid', + 'commit', + '--allow-empty', + '-m', + 'fixture', + ], + { cwd: checkout, stdio: 'ignore' }, + ); + execFileSync('git', ['checkout', '-b', branch], { cwd: checkout, stdio: 'ignore' }); + writeSlot({ graphPath: commitGraph }); + fs.writeFileSync( + path.join(home, 'registry.json'), + JSON.stringify([ + { + name: 'wt', + path: checkout, + storagePath: slot, + indexedAt: '', + lastCommit: '', + branches: [{ branch }], + }, + ]), + ); + const indexDir = path.join(slot, 'branches', branchSlug(branch)); + const graph = `lbug.generation.${'a'.repeat(32)}`; + fs.mkdirSync(indexDir, { recursive: true }); + fs.writeFileSync( + path.join(indexDir, 'gitnexus.json'), + JSON.stringify({ repoPath: checkout, graphPath: graph }), + ); + fs.writeFileSync(path.join(indexDir, graph), 'graph'); + expect(load(file).findRegisteredRepo(checkout)?.lbugPath).toBe(path.join(indexDir, graph)); + }); + + it.each(HOOK_COPIES)('resolves an unregistered local private generation (%s)', (file) => { + const indexDir = path.join(checkout, '.gitnexus'); + const graph = `lbug.generation.${'a'.repeat(32)}`; + fs.mkdirSync(indexDir, { recursive: true }); + fs.writeFileSync( + path.join(indexDir, 'gitnexus.json'), + JSON.stringify({ repoPath: checkout, graphPath: graph }), + ); + fs.writeFileSync(path.join(indexDir, graph), 'graph'); + const hook = load(file) as unknown as { findLocalOwnedRepo: (cwd: string) => HookRepo }; + expect(hook.findLocalOwnedRepo(checkout)?.lbugPath).toBe(path.join(indexDir, graph)); + }); + it.each(HOOK_COPIES)('returns the commit graph a shared slot records (%s)', (file) => { writeSlot({ graphPath: commitGraph }); expect(load(file).findRegisteredRepo(checkout)?.lbugPath).toBe(commitGraph); diff --git a/gitnexus/test/unit/lbug-vector-pool.test.ts b/gitnexus/test/unit/lbug-vector-pool.test.ts new file mode 100644 index 000000000..259140fa6 --- /dev/null +++ b/gitnexus/test/unit/lbug-vector-pool.test.ts @@ -0,0 +1,103 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + fts: vi.fn(), + threads: vi.fn(), + connections: [] as Array<{ close: ReturnType }>, +})); +vi.mock('@ladybugdb/core', () => ({ + default: { + Connection: class { + close = vi.fn(async () => {}); + setMaxNumThreadForExec = mocks.threads; + constructor() { + mocks.connections.push(this); + } + async query() { + return { getAll: async () => [], close: async () => {} }; + } + }, + }, +})); +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + isReadOnlyDbError: () => false, + loadFTSExtension: mocks.fts, + loadVectorExtension: async () => false, +})); +vi.mock('../../src/core/lbug/lbug-config.js', async (original) => ({ + ...(await original()), + createLbugDatabase: mocks.create, +})); +import { closeLbug, initLbug, isLbugReady } from '../../src/core/lbug/pool-adapter.js'; + +let directory: string; +const options = { kind: 'vector' as const, bufferPoolBytes: 64 * 1024 * 1024, threads: 2 }; +async function file(name: string) { + const target = path.join(directory, name); + await fs.writeFile(target, 'fixture'); + return target; +} +beforeEach(async () => { + vi.clearAllMocks(); + mocks.connections.length = 0; + mocks.create.mockImplementation(() => ({ + init: vi.fn(async () => {}), + close: vi.fn(async () => {}), + })); + mocks.fts.mockResolvedValue(true); + mocks.threads.mockResolvedValue(undefined); + directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-pool-')); +}); +afterEach(async () => { + await closeLbug(); + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('vector pool isolation', () => { + it('retains the graph-cache capacity while bounding vector residents independently', async () => { + for (let i = 0; i < 5; i++) await initLbug(`graph-${i}`, await file(`graph-${i}`)); + for (let i = 0; i < 6; i++) await initLbug(`vector-${i}`, await file(`vector-${i}`), options); + for (let i = 0; i < 5; i++) expect(isLbugReady(`graph-${i}`)).toBe(true); + expect(isLbugReady('vector-0')).toBe(false); + for (let i = 1; i < 6; i++) expect(isLbugReady(`vector-${i}`)).toBe(true); + expect(mocks.fts).toHaveBeenCalledTimes(5); + expect(mocks.threads).toHaveBeenCalledTimes(6 * 8); + expect(mocks.create).toHaveBeenLastCalledWith( + expect.anything(), + path.join(directory, 'vector-5'), + { readOnly: true, bufferPoolBytes: options.bufferPoolBytes }, + ); + }); + it('retains the actual reservation when its caller mutates an options object', async () => { + const target = await file('vectors'); + const reservation = { ...options }; + await initLbug('vectors', target, reservation); + reservation.bufferPoolBytes *= 2; + await expect(initLbug('vectors', target, reservation)).rejects.toThrow( + 'different resource reservation', + ); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + it('closes an incompletely initialized native handle and permits a clean retry', async () => { + const target = await file('vectors'); + mocks.threads.mockRejectedValueOnce(new Error('thread configuration failed')); + await expect(initLbug('vectors', target, options)).rejects.toThrow( + 'thread configuration failed', + ); + expect(mocks.create.mock.results[0].value.close).toHaveBeenCalledOnce(); + expect(mocks.connections[0].close).toHaveBeenCalledOnce(); + await initLbug('vectors', target, options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + it('refuses uncheckpointed vector files without quarantining their sidecars', async () => { + const target = await file('vectors'); + await fs.writeFile(target + '.wal', 'pending writes'); + await expect(initLbug('vectors', target, options)).rejects.toThrow('unfinished native writes'); + expect(mocks.create).not.toHaveBeenCalled(); + expect(await fs.readFile(target + '.wal', 'utf8')).toBe('pending writes'); + }); +}); diff --git a/gitnexus/test/unit/local-backend-vector-generation.test.ts b/gitnexus/test/unit/local-backend-vector-generation.test.ts new file mode 100644 index 000000000..4db6b30d2 --- /dev/null +++ b/gitnexus/test/unit/local-backend-vector-generation.test.ts @@ -0,0 +1,79 @@ +import path from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const mocks = vi.hoisted(() => ({ query: vi.fn(), embed: vi.fn(), metadata: vi.fn() })); +vi.mock('../../src/core/lbug/pool-adapter.js', async (original) => ({ + ...(await original()), + executeQuery: mocks.query, +})); +vi.mock('../../src/storage/repo-manager.js', async (original) => ({ + ...(await original()), + loadMeta: mocks.metadata, +})); +vi.mock('../../src/mcp/core/embedder.js', () => ({ + embedQuery: mocks.embed, + getEmbeddingDims: () => 4, +})); +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const generation = 'a'.repeat(32); +const graphPath = path.join('/repo/.gitnexus', `lbug.generation.${generation}`); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: path.basename(graphPath), + graphGeneration: { version: 1, id: generation, coveredContent: 'b'.repeat(64) }, + embeddingJob: { version: 1, identity: 'c'.repeat(64), graphGeneration: generation }, +}; +interface Reader { + semanticSearch( + repo: { lbugPath: string; generationUnknown?: boolean }, + query: string, + limit: number, + ): Promise; + lastObservedPoolState: Map; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.query.mockResolvedValue([]); + mocks.metadata.mockResolvedValue(metadata); +}); + +describe('MCP generation gate', () => { + it.each(['pending', 'foreign-metadata', 'unknown-handle'])( + 'skips model and table access for %s', + async (state) => { + const backend = new LocalBackend() as unknown as Reader; + if (state === 'foreign-metadata') + mocks.metadata.mockResolvedValue({ + ...metadata, + graphPath: `lbug.generation.${'d'.repeat(32)}`, + }); + expect( + await backend.semanticSearch( + { lbugPath: graphPath, generationUnknown: state === 'unknown-handle' }, + 'hello', + 5, + ), + ).toEqual([]); + expect(mocks.query).not.toHaveBeenCalled(); + expect(mocks.embed).not.toHaveBeenCalled(); + }, + ); + it('keeps a warm reader on its own metadata when publication advances', async () => { + const backend = new LocalBackend() as unknown as Reader; + backend.lastObservedPoolState.set(graphPath, { metadata }); + mocks.metadata.mockResolvedValue({ + ...metadata, + graphPath: `lbug.generation.${'d'.repeat(32)}`, + graphGeneration: { ...metadata.graphGeneration, id: 'd'.repeat(32) }, + }); + const repo = { lbugPath: graphPath, generationUnknown: false }; + expect(await backend.semanticSearch(repo, 'hello', 5)).toEqual([]); + expect(repo.generationUnknown).toBe(false); + expect(mocks.query).not.toHaveBeenCalled(); + expect(mocks.embed).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts new file mode 100644 index 000000000..9b1947368 --- /dev/null +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -0,0 +1,99 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + assertOwnedVectorGeneration, + embeddingGenerationDirectory, + selectVectorGeneration, +} from '../../src/storage/vector-generation.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const generation = 'a'.repeat(32); +const identity = 'b'.repeat(64); +const meta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: `lbug.generation.${generation}`, + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind: 'exact', + }, +} satisfies RepoMeta; +const select = (metadata: RepoMeta = meta, graphPath = path.join('/slot', meta.graphPath)) => + selectVectorGeneration('/slot', metadata, graphPath, 10000); +const directories: string[] = []; +afterEach(async () => { + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('vector generation routing', () => { + it('preserves the legacy reader and routes a complete matching artifact', () => { + expect(select({ repoPath: '/repo', lastCommit: '', indexedAt: '' })).toEqual({ + kind: 'legacy', + }); + expect(select()).toEqual({ + kind: 'ready', + path: path.join('/slot', 'embeddings', identity, 'vectors'), + generation: meta.vectorGeneration, + }); + }); + it('gates pending, foreign, missing and oversized exact artifacts', () => { + expect( + select({ + ...meta, + vectorGeneration: undefined, + embeddingJob: { version: 1, identity, graphGeneration: generation }, + }), + ).toMatchObject({ kind: 'unavailable', reason: 'embedding-pending' }); + expect(select({ ...meta, graphGeneration: undefined })).toMatchObject({ kind: 'unavailable' }); + expect(select(meta, '/slot/lbug')).toMatchObject({ kind: 'unavailable' }); + expect( + select({ + ...meta, + vectorGeneration: { ...meta.vectorGeneration, graphGeneration: 'f'.repeat(32) }, + }), + ).toMatchObject({ kind: 'unavailable' }); + expect( + select({ ...meta, vectorGeneration: { ...meta.vectorGeneration, chunks: 10001 } }), + ).toMatchObject({ kind: 'unavailable', reason: 'vector-index-pending' }); + expect( + select({ + ...meta, + embeddingJob: { version: 1, identity: 'f'.repeat(64), graphGeneration: generation }, + }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it.each([NaN, -1, 0.5, Infinity])('rejects invalid coverage %s', (chunks) => { + expect( + select({ ...meta, vectorGeneration: { ...meta.vectorGeneration, chunks } }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it('rejects traversal and every symlink boundary', async () => { + expect(() => embeddingGenerationDirectory('/slot', '../foreign')).toThrow('identity'); + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-path-')); + directories.push(root); + const job = embeddingGenerationDirectory(root, identity); + await fs.mkdir(job, { recursive: true }); + const file = path.join(job, 'vectors'); + await fs.writeFile(file, 'artifact'); + expect(await assertOwnedVectorGeneration(root, identity)).toBe(file); + await fs.rename(file, file + '.old'); + await fs.symlink(file + '.old', file); + await expect(assertOwnedVectorGeneration(root, identity)).rejects.toThrow('regular file'); + await fs.unlink(file); + await fs.rename(job, job + '.old'); + await fs.symlink(job + '.old', job, 'dir'); + await expect(assertOwnedVectorGeneration(root, identity)).rejects.toThrow('directory'); + }); +}); From 621bd4e20f3f0c51ca34eb8ba4aedbd4fdbe0bcc Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 14:10:29 +0000 Subject: [PATCH 07/67] feat(embeddings): coordinate durable jobs Join published graph selection, durable work, bounded native handles and vector publication behind one inactive controller. Limit optional HTTP response decoding within the job reservation and admit supported large inputs under larger budgets. Refs #3598 --- gitnexus/src/core/analysis-resource-budget.ts | 4 +- .../core/embeddings/embedding-job-identity.ts | 2 + .../core/embeddings/embedding-job-runner.ts | 14 +- gitnexus/src/core/embeddings/embedding-job.ts | 341 +++++++++++++++++ gitnexus/src/core/embeddings/http-client.ts | 50 ++- .../embedding-job-controller.test.ts | 358 ++++++++++++++++++ .../unit/analysis-resource-budget.test.ts | 11 + .../test/unit/embedding-job-journal.test.ts | 1 + .../http-embedding-response-budget.test.ts | 110 ++++++ 9 files changed, 884 insertions(+), 7 deletions(-) create mode 100644 gitnexus/src/core/embeddings/embedding-job.ts create mode 100644 gitnexus/test/integration/embedding-job-controller.test.ts create mode 100644 gitnexus/test/unit/http-embedding-response-budget.test.ts diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index f796b4fb2..c02da1df6 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -83,7 +83,9 @@ export function resolveAnalysisResourceBudget( // Zero is the native engine's unbounded override, never a bounded allocation. const graphPoolMb = Math.ceil(graphPoolBytes / MIB); const modelMb = integer(request.modelMemoryMb, 'model reservation', 0); - const queueMb = Math.max(16, Math.min(128, Math.floor(totalMb / 32))); + // A supported 10 MiB source node needs its full text/AST working set. Larger + // explicit budgets must be able to admit it without truncating its content. + const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32))); const embeddingFixedMb = headroomMb + graphPoolMb + VECTOR_POOL_MB + modelMb + queueMb; const includeParsing = request.includeParsing !== false; const parseFixedMb = diff --git a/gitnexus/src/core/embeddings/embedding-job-identity.ts b/gitnexus/src/core/embeddings/embedding-job-identity.ts index 49f8cf905..a3c7bec5d 100644 --- a/gitnexus/src/core/embeddings/embedding-job-identity.ts +++ b/gitnexus/src/core/embeddings/embedding-job-identity.ts @@ -9,6 +9,7 @@ export interface EmbeddingGraphIdentity { analyzer: string; parser: string; schema: string; + vectorSchema: string; } const digest = (value: unknown): string => @@ -31,6 +32,7 @@ export function embeddingJobIdentity( graph.analyzer, graph.parser, graph.schema, + graph.vectorSchema, ]; if ( source.some((value) => typeof value !== 'string' || !value) || diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index fd0ab7cd3..c8018e591 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -52,10 +52,12 @@ export async function runEmbeddingJob( const config = resolveEmbeddingConfig(options.config ?? {}); const budget = new EmbeddingByteBudget(options.queueBytes); const dimensions = store.options.dimensions; - // UTF-16 text + worst-case JSON escaping/encoding + float response, JS array - // and native copies. The provider's runtime has a separate reservation. + // UTF-16 text + worst-case JSON escaping/encoding, bounded HTTP response + // decoding/JSON parsing, float results, JS arrays and native copies. HTTP + // responses are capped at 4096 + 32*dimensions bytes per row by the controller. + // The provider's runtime has a separate reservation. const rowBytes = (textChars: number, idChars: number) => - 16 * textChars + 32 * dimensions + 1024 + 4 * idChars; + 16 * textChars + 512 * dimensions + 64 * 1024 + 4 * idChars; const minimumBatchBytes = rowBytes(config.maxInputChars, 0); if (minimumBatchBytes > budget.limit) throw new Error('Embedding input/vector batch does not fit the resource budget'); @@ -220,14 +222,16 @@ export async function runEmbeddingJob( reuse.options.batchBytes, store.options.batchBytes, budget.limit - budget.usedBytes, - ) / rowBytes(0, node.id.length), + ) / + (32 * dimensions + 1024 + 4 * node.id.length), ), ); if (copyRows < 1) throw new Error('Reusable vector batch does not fit the resource budget'); for (let start = 0; start < reusable.chunks; start += copyRows) { const length = Math.min(copyRows, reusable.chunks - start); - const release = budget.reserve(length * rowBytes(0, node.id.length)); + // Reuse has no HTTP body or JSON decoder; retain only vector copies. + const release = budget.reserve(length * (32 * dimensions + 1024 + 4 * node.id.length)); try { const chunks = await reuse.readChunks(reusable, start, length); const delta = input.sourceStartLine - (reusable.sourceStartLine ?? 0); diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts new file mode 100644 index 000000000..ed4bc9fd5 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -0,0 +1,341 @@ +/** Common lifecycle for jobs on a published graph. The caller holds its index lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; +import { + EmbeddingJobJournal, + type EmbeddingJobState, +} from '../../storage/embedding-job-journal.js'; +import { + assertOwnedVectorGeneration, + metadataMatchesGraph, +} from '../../storage/vector-generation.js'; +import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; +import { syncPublishedMetadata } from '../analysis-publication.js'; +import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; +import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; +import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; +import { embeddingJobIdentity } from './embedding-job-identity.js'; +import { resolveEmbeddingIdentity, type EmbeddingIdentity } from './embedding-identity.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { + createEmbeddingWorkManifest, + loadEmbeddingWorkManifest, + type EmbeddingManifestSource, +} from './embedding-work-manifest.js'; +import { runEmbeddingJob } from './embedding-job-runner.js'; +import { publishVectorGeneration, type VectorPublicationActions } from './vector-publication.js'; +import type { EmbeddingConfig, ModelProgressCallback } from './types.js'; +import type { embedBatch } from './embedder.js'; + +type GraphSource = EmbeddingManifestSource & { close(): Promise }; +export interface PublishedEmbeddingJobOptions { + metadataDirectory: string; + /** Already resolved/owned graph, pinned for the entire job by the index lock. */ + graphPath: string; + resources: AnalysisResourceBudget['embedding']; + config?: Partial; + signal?: AbortSignal; + onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void; + onModelProgress?: ModelProgressCallback; + log?: (message: string) => void; + metadata: VectorPublicationActions; +} + +/** Provider/source seams allow deterministic native-storage restart tests. */ +export interface PublishedEmbeddingJobDependencies { + identity?: () => EmbeddingIdentity; + openSource?: (file: string, poolBytes: number, threads: number) => Promise; + provider?: typeof embedBatch; + initializeProvider?: (config: EmbeddingConfig) => Promise; +} + +export interface PublishedEmbeddingJobResult { + metadata: RepoMeta; + alreadyComplete: boolean; + phase: 'embedding' | 'indexing' | 'ready'; + state?: EmbeddingJobState; + peakQueueBytes: number; +} + +const digest = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); + +async function existsOwnedFile(file: string): Promise { + try { + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Embedding artifact must be an owned regular file'); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } +} + +async function jobDirectory(root: string, identity: string): Promise { + let directory = await embeddingJobDirectory(root); + for (const component of ['embeddings', identity]) { + const next = path.join(directory, component); + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + +/** A preflight interrupted before any native/job artifact owns no paid work. + * Only its known draft selection files may be removed. Corrupt paid receipts + * remain evidence and must never turn into a fresh, apparently empty job. + */ +async function clearUnstartedManifest(directory: string): Promise { + if ( + (await existsOwnedFile(path.join(directory, 'vectors'))) || + (await existsOwnedFile(path.join(directory, 'job.json'))) + ) + throw new Error('Embedding manifest is missing for existing paid work; artifacts retained'); + const entries = await fs.opendir(directory); + for await (const entry of entries) { + if (/^(manifest\.pending|manifest\.jsonl|keys-\d+\.jsonl)$/.test(entry.name)) { + const file = path.join(directory, entry.name); + await existsOwnedFile(file); + await fs.unlink(file); + } + } + await syncEmbeddingJobDirectory(directory); +} + +export async function runPublishedEmbeddingJob( + options: PublishedEmbeddingJobOptions, + dependencies: PublishedEmbeddingJobDependencies = {}, +): Promise { + const { resources, signal } = options; + signal?.throwIfAborted(); + const root = await embeddingJobDirectory(options.metadataDirectory); + let metadata = await options.metadata.loadMetadata(); + const graph = metadata?.graphGeneration; + const runner = metadata?.runnerIdentity; + if ( + !metadata || + !graph || + graph.version !== 1 || + !/^[a-f0-9]{32}$/.test(graph.id) || + !/^[a-f0-9]{64}$/.test(graph.coveredContent) || + !runner || + runner.schemaVersion !== 4 || + !metadata.parseCacheIdentity || + !metadata.schemaFingerprint || + metadata.incrementalInProgress || + metadata.graphWriteCollapsed || + !metadataMatchesGraph(root, metadata, options.graphPath) + ) + throw new Error('Embedding job requires a verified published graph generation'); + const config = resolveEmbeddingConfig(options.config); + const resolvedVector = (dependencies.identity ?? resolveEmbeddingIdentity)(); + const vector = + dependencies.identity || resolvedVector.provider !== 'local' + ? resolvedVector + : { ...resolvedVector, model: config.modelId }; + if (config.threads > resources.nativeThreads) + throw new Error('Embedding threads exceed the admitted resource budget'); + for (const value of [resources.graphPoolBytes, resources.vectorPoolBytes, resources.queueBytes]) + if (!Number.isSafeInteger(value) || value < 8 * 1024 * 1024) + throw new Error('Invalid embedding resource reservation'); + if (!Number.isSafeInteger(resources.nativeThreads) || resources.nativeThreads < 1) + throw new Error('Invalid embedding CPU reservation'); + const identities = embeddingJobIdentity( + { + generation: graph.id, + coveredContent: graph.coveredContent, + revision: metadata.lastCommit || 'non-git', + // Entrypoint paths are diagnostic. Identical artifacts used by analyze, + // sync and the server must resolve to the same paid job identity. + analyzer: digest([ + runner.schemaVersion, + runner.runtime.version, + runner.runtime.platform, + runner.runtime.architecture, + runner.runtime.modulesAbi, + runner.runtime.libc, + runner.cliVersion, + runner.build.digest, + runner.dependencyRuntime.digest, + ]), + parser: digest(metadata.parseCacheIdentity), + schema: metadata.schemaFingerprint, + vectorSchema: VECTOR_STORE_SCHEMA, + }, + vector, + EMBEDDING_TEXT_VERSION, + config, + ); + const previous = metadata.vectorGeneration; + if (previous?.identity === identities.identity && !metadata.embeddingJob) { + await assertOwnedVectorGeneration(root, previous.identity); + return { metadata, alreadyComplete: true, phase: 'ready', peakQueueBytes: 0 }; + } + if (metadata.embeddingJob && metadata.embeddingJob.identity !== identities.identity) { + options.log?.( + metadata.embeddingJob.graphGeneration !== graph.id + ? 'Embedding graph generation changed; prior paid artifacts retained for compatible reuse.' + : 'Embedding configuration changed; prior paid artifacts retained.', + ); + } else if (previous && previous.vectorIdentity !== identities.vectorIdentity) { + options.log?.( + 'Embedding provider, model or dimensions changed; starting a separate vector space.', + ); + } else if (previous && previous.inputIdentity !== identities.inputIdentity) { + options.log?.('Embedding text options changed; prior vectors are not compatible.'); + } + const directory = await jobDirectory(root, identities.identity); + if (metadata.embeddingJob?.identity !== identities.identity) { + metadata = { + ...metadata, + embeddingJob: { + version: 1, + identity: identities.identity, + graphGeneration: graph.id, + }, + }; + await options.metadata.saveMetadata(metadata); + await syncPublishedMetadata(root, metadata); + } + let source: GraphSource | undefined; + let store: VectorStore | undefined; + let reuse: VectorStore | undefined; + const reusable = + previous && + previous.identity !== identities.identity && + previous.vectorIdentity === identities.vectorIdentity && + previous.inputIdentity === identities.inputIdentity && + previous.dimensions === vector.dimensions; + // A reuse reader shares the vector reservation; it is not a third default pool. + const poolBytes = Math.floor(resources.vectorPoolBytes / (reusable ? 2 : 1)); + const storeResources = { + bufferPoolBytes: poolBytes, + threads: resources.nativeThreads, + batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), + }; + try { + const manifestExists = await existsOwnedFile(path.join(directory, 'manifest.json')); + if (!manifestExists) await clearUnstartedManifest(directory); + const getSource = async (): Promise => + (source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)( + options.graphPath, + resources.graphPoolBytes, + resources.nativeThreads, + )); + const manifest = manifestExists + ? await loadEmbeddingWorkManifest(directory, identities.sourceIdentity) + : await createEmbeddingWorkManifest(directory, identities.sourceIdentity, await getSource()); + const hasStore = await existsOwnedFile(path.join(directory, 'vectors')); + if (!hasStore && (await existsOwnedFile(path.join(directory, 'job.json')))) + throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); + store = await VectorStore.open({ + directory, + identity: identities.identity, + dimensions: vector.dimensions, + mode: hasStore ? 'resume' : 'create', + ...storeResources, + }); + const journal = await EmbeddingJobJournal.recover(directory, identities.identity, manifest); + if (reusable) { + const file = await assertOwnedVectorGeneration(root, previous.identity); + reuse = await VectorStore.open({ + directory: path.dirname(file), + identity: previous.identity, + dimensions: vector.dimensions, + mode: 'read', + ...storeResources, + }); + } + const provider: typeof embedBatch = + dependencies.provider ?? + (async (texts, request) => { + const { embedBatch } = await import('./embedder.js'); + return embedBatch(texts, { + ...request, + maxResponseBytes: texts.length * (4096 + 32 * vector.dimensions), + }); + }); + const initializeProvider = async () => { + if (dependencies.initializeProvider) return dependencies.initializeProvider(config); + if (vector.provider !== 'local') return; + const { initEmbedder } = await import('./embedder.js'); + await initEmbedder(options.onModelProgress, config); + }; + const result = await runEmbeddingJob({ + directory, + manifest, + journal, + store, + reuse, + config, + queueBytes: resources.queueBytes, + // Opening the graph is lazy even on resume: indexing-only retries do not + // enumerate/fetch source or initialize a provider. + source: { + scan: async function* (label) { + yield* (await getSource()).scan(label); + }, + fetch: async (label, ids) => (await getSource()).fetch(label, ids), + }, + provider, + initializeProvider, + signal, + onProgress: options.onProgress, + }); + await store.close(); + store = undefined; + await reuse?.close(); + reuse = undefined; + await source?.close(); + source = undefined; + signal?.throwIfAborted(); + if (!result.indexKind) + return { metadata, alreadyComplete: false, phase: 'embedding', ...result }; + const published = await publishVectorGeneration( + root, + { + version: 1, + identity: identities.identity, + graphGeneration: graph.id, + vectorIdentity: identities.vectorIdentity, + inputIdentity: identities.inputIdentity, + dimensions: vector.dimensions, + nodes: manifest.totalNodes, + chunks: result.state.chunks, + indexKind: result.indexKind, + }, + storeResources, + options.metadata, + ); + if (!published) return { metadata, alreadyComplete: false, phase: 'indexing', ...result }; + if (journal.snapshot().phase !== 'ready') await journal.ready(); + options.onProgress?.(journal.snapshot(), result.peakQueueBytes); + return { + metadata: published, + alreadyComplete: false, + phase: 'ready', + state: journal.snapshot(), + peakQueueBytes: result.peakQueueBytes, + }; + } finally { + // Sequential finally blocks close every native handle even when one close fails. + try { + await store?.close(); + } finally { + try { + await reuse?.close(); + } finally { + await source?.close(); + } + } + } +} diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 689d3cf1d..946881361 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -46,6 +46,40 @@ interface HttpConfig { export interface EmbeddingRequestOptions { signal?: AbortSignal; + /** Encoded successful response bound, reserved by the calling job. */ + maxResponseBytes?: number; +} + +class EmbeddingResponseBudgetError extends Error {} + +/** Bound encoded bytes before JSON decoding; Content-Length alone is untrusted. */ +async function readEmbeddingResponse(response: Response, maximum?: number): Promise { + if (maximum === undefined) return response.json(); + if (!Number.isSafeInteger(maximum) || maximum < 1) + throw new Error('Invalid embedding response byte limit'); + const reader = response.body?.getReader(); + if (!reader) throw new Error('Embedding response has no body'); + let complete = false; + try { + const declared = response.headers.get('content-length'); + if (declared !== null && Number(declared) > maximum) + throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget'); + const bytes = Buffer.allocUnsafe(maximum); + let size = 0; + for (;;) { + const next = await reader.read(); + if (next.done) break; + if (next.value.byteLength > maximum - size) + throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget'); + bytes.set(next.value, size); + size += next.value.byteLength; + } + complete = true; + return JSON.parse(new TextDecoder().decode(bytes.subarray(0, size))); + } finally { + if (!complete) await reader.cancel().catch(() => {}); + reader.releaseLock(); + } } let lastHttpRequestStartedAt: number | undefined; @@ -427,6 +461,11 @@ const httpEmbedBatch = async ( timeoutMs = DEFAULT_HTTP_TIMEOUT_MS, retryTimeouts = false, ): Promise => { + if ( + requestOptions.maxResponseBytes !== undefined && + (!Number.isSafeInteger(requestOptions.maxResponseBytes) || requestOptions.maxResponseBytes < 1) + ) + throw new Error('Invalid embedding response byte limit'); const requestBody: { input: string[]; model: string; dimensions?: number } = { input: batch, model, @@ -490,8 +529,17 @@ const httpEmbedBatch = async ( // and the same breaker accounting as any other endpoint fault (#2790). let payload: { data: EmbeddingItem[] }; try { - payload = (await attemptResp.json()) as { data: EmbeddingItem[] }; + payload = (await readEmbeddingResponse( + attemptResp, + requestOptions.maxResponseBytes, + )) as { + data: EmbeddingItem[]; + }; } catch (err) { + if (err instanceof EmbeddingResponseBudgetError) + throw new RetryableEmbeddingBodyError( + `Embedding endpoint exceeded its ${requestOptions.maxResponseBytes}-byte response budget (${safeUrl(url)}, batch ${batchIndex})`, + ); // Not every `.json()` rejection is a parse error: the per-attempt // signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is // wired to the body stream, so a stalled body rejects with the abort diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts new file mode 100644 index 000000000..3067715b0 --- /dev/null +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -0,0 +1,358 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { runPublishedEmbeddingJob } from '../../src/core/embeddings/embedding-job.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; + +let temporary: TestDBHandle | undefined; +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await temporary?.cleanup(); + temporary = undefined; +}); + +async function fixture(count = 130) { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const root = temporary.dbPath; + let generation = 1; + const makeMeta = (): RepoMeta => ({ + repoPath: root, + lastCommit: 'commit', + indexedAt: new Date().toISOString(), + graphPath: `lbug.generation.${String(generation).padStart(32, '0')}`, + graphGeneration: { + version: 1, + id: String(generation).padStart(32, '0'), + coveredContent: 'a'.repeat(64), + }, + schemaFingerprint: 'graph-schema', + parseCacheIdentity: { base: 'parser' }, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'test', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: '127', + libc: 'glibc', + }, + invokedArtifact: { path: 'analyze', digest: 'cli' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'build', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'runtime', + }, + }, + scopeExtractionReceipt: 1, + }); + await saveMeta(root, makeMeta()); + const node = (id: string): EmbeddableNode => ({ + id, + name: id.split(':').at(-1) ?? id, + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, + }); + const scan = vi.fn(async function* (label: string) { + if (label !== 'Const') return; + for (let i = 0; i < count; i++) + yield { + id: `Const:a.ts:n${String(i).padStart(5, '0')}`, + contentBytes: Buffer.byteLength('const value = 1;'), + metadataBytes: 1024, + }; + }); + const fetch = vi.fn(async (_label: string, ids: readonly string[]) => ids.map(node)); + const close = vi.fn(async () => {}); + const openSource = vi.fn(async () => ({ scan, fetch, close })); + const initializeProvider = vi.fn(async () => {}); + const provider = vi.fn(async (texts: string[]) => { + expect((await loadMeta(root))?.embeddingJob).toBeDefined(); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }); + const identity = vi.fn(() => ({ provider: 'local', model: 'test', dimensions: 4 })); + const resources = { + graphPoolBytes: 16 * 1024 * 1024, + vectorPoolBytes: 64 * 1024 * 1024, + modelBytes: 0, + queueBytes: 16 * 1024 * 1024, + nativeThreads: 1, + reservedBytes: 96 * 1024 * 1024, + }; + const metadata = { + loadMetadata: () => loadMeta(root), + saveMetadata: (meta: RepoMeta) => saveMeta(root, meta), + }; + const options = () => ({ + metadataDirectory: root, + graphPath: path.join(root, `lbug.generation.${String(generation).padStart(32, '0')}`), + resources, + config: { threads: 1, subBatchSize: 8 }, + metadata, + }); + const dependencies = { openSource, provider, initializeProvider, identity }; + const run = (extra: Partial[0]> = {}) => + runPublishedEmbeddingJob({ ...options(), ...extra }, dependencies); + return { + root, + run, + options, + dependencies, + provider, + initializeProvider, + openSource, + scan, + fetch, + close, + nextGeneration: async () => { + const prior = await loadMeta(root); + generation++; + await saveMeta(root, { ...makeMeta(), vectorGeneration: prior?.vectorGeneration }); + }, + }; +} + +describe('published graph embedding job controller', () => { + it('publishes complete vectors, then takes a no-model/no-writer fast path', async () => { + const f = await fixture(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.nodes).toBe(130); + expect(result.metadata.embeddingJob).toBeUndefined(); + expect(result.state?.completedUnits).toBe(3); + expect(f.initializeProvider).toHaveBeenCalledOnce(); + f.provider.mockClear(); + f.openSource.mockClear(); + const open = vi.spyOn(VectorStore, 'open'); + expect((await f.run()).alreadyComplete).toBe(true); + expect(open).not.toHaveBeenCalled(); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('resumes acknowledged work with a smaller batch and queue without rescanning keys', async () => { + const f = await fixture(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits === 1) throw new Error('stop after acknowledgement'); + }, + }), + ).rejects.toThrow('stop after acknowledgement'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(64); + f.provider.mockClear(); + f.scan.mockClear(); + const result = await f.run({ + config: { threads: 1, subBatchSize: 1 }, + resources: { ...f.options().resources, queueBytes: 8 * 1024 * 1024 }, + }); + expect(result.phase).toBe('ready'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + expect(f.scan).not.toHaveBeenCalled(); + expect(result.peakQueueBytes).toBeLessThanOrEqual(8 * 1024 * 1024); + }); + + it('retries pointer publication without source or provider work', async () => { + const f = await fixture(5); + const metadata = f.options().metadata; + await expect( + f.run({ + metadata: { + ...metadata, + saveMetadata: async (meta) => { + if (meta.vectorGeneration) throw new Error('disk unavailable for pointer'); + await metadata.saveMetadata(meta); + }, + }, + }), + ).rejects.toThrow('disk unavailable'); + f.provider.mockClear(); + f.openSource.mockClear(); + f.initializeProvider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + }); + + it('copies compatible paid vectors to the next graph with no provider call', async () => { + const f = await fixture(5); + const first = await f.run(); + await f.nextGeneration(); + f.provider.mockClear(); + f.initializeProvider.mockClear(); + const next = await f.run(); + expect(next.phase).toBe('ready'); + expect(next.metadata.vectorGeneration?.identity).not.toBe( + first.metadata.vectorGeneration?.identity, + ); + expect(next.state?.reusedNodes).toBe(5); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + if (!first.metadata.vectorGeneration) throw new Error('Expected published vectors'); + expect( + await fs.stat( + path.join(f.root, 'embeddings', first.metadata.vectorGeneration.identity, 'vectors'), + ), + ).toBeDefined(); + }); + + it('does no paid replay if the process stops after publishing the vector pointer', async () => { + const f = await fixture(5); + const ready = vi + .spyOn(EmbeddingJobJournal.prototype, 'ready') + .mockRejectedValueOnce(new Error('stopped after pointer')); + await expect(f.run()).rejects.toThrow('stopped after pointer'); + expect((await loadMeta(f.root))?.vectorGeneration).toBeDefined(); + f.provider.mockClear(); + f.openSource.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + ready.mockRestore(); + }); + + it('recovers an interrupted manifest preflight before any provider or vector mutation', async () => { + const f = await fixture(5); + f.scan.mockImplementationOnce(async function* () { + throw new Error('preflight stopped'); + }); + await expect(f.run()).rejects.toThrow('preflight stopped'); + expect(f.provider).not.toHaveBeenCalled(); + expect((await f.run()).phase).toBe('ready'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(5); + }); + + it('publishes empty coverage without a model request', async () => { + const f = await fixture(0); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.chunks).toBe(0); + expect(result.metadata.capabilities?.vectorSearch.status).toBe('unavailable'); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('admits a 10 MiB fallback file and embeds its final marker without truncation', async () => { + const f = await fixture(0); + const content = 'a'.repeat(10 * 1024 * 1024 - 30) + '\nFINAL_UNTRUNCATED_MARKER'; + const id = 'File:large.txt'; + let sawEnd = false; + let requests = 0; + const result = await runPublishedEmbeddingJob( + { + ...f.options(), + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 16 * 1024 ** 3, + availableCpus: 4, + totalMb: 8192, + modelMemoryMb: 0, + includeParsing: false, + }).embedding, + }, + { + ...f.dependencies, + openSource: async () => ({ + async *scan(label) { + if (label === 'File') + yield { id, contentBytes: Buffer.byteLength(content), metadataBytes: 1024 }; + }, + fetch: async () => [ + { + id, + label: 'File', + name: 'large.txt', + filePath: 'large.txt', + content, + startLine: 1, + endLine: 2, + }, + ], + close: async () => {}, + }), + // Count inputs without retaining the complete test corpus in spy history. + provider: async (texts) => { + requests++; + sawEnd ||= texts.some((text) => text.includes('FINAL_UNTRUNCATED_MARKER')); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }, + }, + ); + expect(result.phase).toBe('ready'); + expect(sawEnd).toBe(true); + expect(requests).toBeGreaterThan(100); + expect(result.peakQueueBytes).toBeLessThanOrEqual(256 * 1024 * 1024); + }, 60000); + + it('reconstructs a corrupt journal only from verified native complete groups', async () => { + const f = await fixture(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits === 1) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const meta = await loadMeta(f.root); + if (!meta?.embeddingJob) throw new Error('Expected pending job'); + const job = path.join(f.root, 'embeddings', meta.embeddingJob.identity); + await fs.writeFile(path.join(job, 'job.json'), '{broken'); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(64); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + expect((await fs.readdir(job)).some((name) => name.startsWith('job.json.preserved-'))).toBe( + true, + ); + }); + + it('rejects a missing paid artifact before model or fresh-store creation', async () => { + const f = await fixture(5); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const meta = await loadMeta(f.root); + if (!meta?.embeddingJob) throw new Error('Expected pending job'); + const job = path.join(f.root, 'embeddings', meta.embeddingJob.identity); + await fs.rename(path.join(job, 'vectors'), path.join(job, 'vectors.retained')); + f.provider.mockClear(); + await expect(f.run()).rejects.toThrow('vectors are missing'); + expect(f.provider).not.toHaveBeenCalled(); + await expect(fs.stat(path.join(job, 'vectors'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('refuses a symlink job root before writing metadata or requesting vectors', async () => { + const f = await fixture(5); + await fs.mkdir(path.join(f.root, 'other')); + await fs.symlink(path.join(f.root, 'other'), path.join(f.root, 'embeddings'), 'dir'); + await expect(f.run()).rejects.toThrow('owned directory'); + expect((await loadMeta(f.root))?.embeddingJob).toBeUndefined(); + expect(f.provider).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 4c5f5c7d3..238d0fcc3 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -49,6 +49,17 @@ describe('analysis resource admission', () => { expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(1024 * MIB); }); + it('admits the full working set of a supported 10 MiB node with a larger budget', () => { + const budget = resolveAnalysisResourceBudget({ + ...request, + totalMb: 8192, + modelMemoryMb: 0, + includeParsing: false, + }); + expect(budget.embedding.queueBytes).toBeGreaterThan(18 * 10 * MIB + MIB); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + }); + it('rejects impossible explicit combinations before work starts', () => { expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 128 })).toThrow( 'cannot admit', diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index ae0c1bd16..b8706a3f3 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -209,6 +209,7 @@ describe('embedding job identity', () => { analyzer: 'analyzer', parser: 'parser', schema: 'schema', + vectorSchema: 'vectors-v2', }; const vector = { provider: 'local', model: 'model', dimensions: 384 }; const config = { diff --git a/gitnexus/test/unit/http-embedding-response-budget.test.ts b/gitnexus/test/unit/http-embedding-response-budget.test.ts new file mode 100644 index 000000000..e6f2c87a7 --- /dev/null +++ b/gitnexus/test/unit/http-embedding-response-budget.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.resetModules(); +}); + +async function client() { + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'https://budget.example/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'test-model'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '1'); + return (await import('../../src/core/embeddings/http-client.js')).httpEmbed; +} + +describe('bounded HTTP embedding responses', () => { + it('preserves vectors when the JSON arrives across one-byte UTF-8 chunks', async () => { + const embed = await client(); + const body = Buffer.from( + JSON.stringify({ data: [{ embedding: [0.25, 0.5, 0.75, 1] }], model: '漢😀' }), + ); + vi.stubGlobal( + 'fetch', + async () => + new Response( + new ReadableStream({ + start(controller) { + for (const byte of body) controller.enqueue(new Uint8Array([byte])); + controller.close(); + }, + }), + ), + ); + const result = await embed(['text'], { maxResponseBytes: body.byteLength }); + expect(Array.from(result[0])).toEqual([0.25, 0.5, 0.75, 1]); + }); + + it.each([undefined, '1'])( + 'cancels an oversized stream with content-length %s', + async (length) => { + const embed = await client(); + const cancel = vi.fn(); + let pulls = 0; + vi.stubGlobal( + 'fetch', + async () => + new Response( + new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array(32)); + }, + cancel, + }), + { headers: length === undefined ? {} : { 'content-length': length } }, + ), + ); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(cancel).toHaveBeenCalledOnce(); + expect(pulls).toBeLessThanOrEqual(4); + }, + ); + + it('refuses an oversized declared body before reading or decoding it', async () => { + const embed = await client(); + const cancel = vi.fn(); + vi.stubGlobal( + 'fetch', + async () => + new Response(new ReadableStream({ cancel }), { + headers: { 'content-length': '1000000000' }, + }), + ); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(cancel).toHaveBeenCalledOnce(); + }); + + it('retains the existing body retry behavior under the byte limit', async () => { + const embed = await client(); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '2'); + vi.stubEnv('GITNEXUS_EMBEDDING_RETRY_CAP_MS', '1'); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response('{"data":')) + .mockResolvedValueOnce(new Response(JSON.stringify({ data: [{ embedding: [1, 0, 0, 0] }] }))); + vi.stubGlobal('fetch', fetch); + expect(Array.from((await embed(['text'], { maxResponseBytes: 1024 }))[0])).toEqual([ + 1, 0, 0, 0, + ]); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it.each([0, -1, NaN, Infinity, 1.5])( + 'rejects invalid budget %s before fetching', + async (maximum) => { + const embed = await client(); + const fetch = vi.fn(); + vi.stubGlobal('fetch', fetch); + await expect(embed(['text'], { maxResponseBytes: maximum })).rejects.toThrow( + 'Invalid embedding response', + ); + expect(fetch).not.toHaveBeenCalled(); + }, + ); +}); From 5bea9f4f7721d8a36bb6a930ab6f857e3f241b90 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 14:53:10 +0000 Subject: [PATCH 08/67] feat(analyze): pass phase resource allocations Carry per-run worker heaps and native pools without changing legacy defaults. Hold parser reservations through safe native exit; refuse database reallocation after an unverified close. --- gitnexus/src/core/analysis-resource-budget.ts | 11 +- .../ingestion/pipeline-phases/parse-impl.ts | 36 ++++- gitnexus/src/core/ingestion/pipeline.ts | 2 + .../src/core/ingestion/workers/worker-pool.ts | 55 ++++++-- gitnexus/src/core/lbug/lbug-adapter.ts | 82 +++++++++--- gitnexus/src/core/run-analyze.ts | 13 +- .../unit/analysis-resource-budget.test.ts | 10 ++ .../test/unit/lbug-analysis-resources.test.ts | 126 ++++++++++++++++++ .../unit/parse-impl-worker-lazy-cache.test.ts | 64 +++++++++ .../unit/worker-pool-resource-limits.test.ts | 31 +++++ .../unit/worker-pool-timeout-retire.test.ts | 65 +++++++++ 11 files changed, 465 insertions(+), 30 deletions(-) create mode 100644 gitnexus/test/unit/lbug-analysis-resources.test.ts diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index c02da1df6..5cd86c508 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -34,6 +34,8 @@ export interface AnalysisResourceBudget { readonly workerHeapMb: number; readonly workerNativeBytes: number; readonly graphPoolBytes: number; + /** Graph writes run after parser workers drain and may use the full CPU allowance. */ + readonly nativeThreads: number; readonly reservedBytes: number; }; readonly embedding: { @@ -95,10 +97,12 @@ export function resolveAnalysisResourceBudget( request.mainHeapMb === undefined ? Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) : integer(request.mainHeapMb, 'main heap reservation'); - if (mainHeapMb < MIN_MAIN_HEAP_MB || mainHeapMb > maximumMainMb) { + // --memory-budget has an established 200 MiB minimum. Preserve that explicit + // choice; the larger floor is only the automatic sizing policy. + const minimumMainMb = request.mainHeapMb === undefined ? MIN_MAIN_HEAP_MB : 200; + if (mainHeapMb < minimumMainMb || mainHeapMb > maximumMainMb) { const minimum = - Math.max(MIN_MAIN_HEAP_MB, request.mainHeapMb ?? 0) + - Math.max(embeddingFixedMb, parseFixedMb); + Math.max(minimumMainMb, request.mainHeapMb ?? 0) + Math.max(embeddingFixedMb, parseFixedMb); throw new Error( `Resource budget ${totalMb} MiB cannot admit parser/model work; requires at least ${minimum} MiB for these allocations`, ); @@ -161,6 +165,7 @@ export function resolveAnalysisResourceBudget( workerHeapMb, workerNativeBytes: bytes(WORKER_NATIVE_MB, 'worker native reservation'), graphPoolBytes, + nativeThreads: cpus, reservedBytes: bytes(parseReservedMb, 'parse reservation'), }), embedding: Object.freeze({ diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index a93a4b0a7..fb872edb5 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -616,10 +616,22 @@ export async function runChunkedParseAndResolve( // shrink an incremental re-analyze: `totalParseable` counts every parseable // file in the scan, not the changed ones, so a warm run of a large repo still // spawns the full requested pool. - const effectivePoolSize = + const workerResources = options?.workerResources; + if ( + workerResources && + (!Number.isSafeInteger(workerResources.workers) || + workerResources.workers < 1 || + !Number.isSafeInteger(workerResources.workerHeapMb) || + workerResources.workerHeapMb < 512 || + (explicitPoolSize !== undefined && explicitPoolSize > workerResources.workers)) + ) + throw new Error('Parser worker configuration exceeds its admitted resource budget'); + const effectivePoolSize = Math.min( explicitPoolSize && explicitPoolSize > 0 ? Math.min(explicitPoolSize, Math.max(1, totalParseable)) - : Math.min(resolveAutoPoolSize(), workProportionalCap); + : Math.min(resolveAutoPoolSize(), workProportionalCap), + workerResources?.workers ?? Infinity, + ); // Deliberate over-subscription is the operator's call, so this warns rather // than caps — silently capping is what the override exists to stop. But an // exported `GITNEXUS_WORKER_POOL_SIZE` applies to EVERY analyze in a @@ -725,6 +737,9 @@ export async function runChunkedParseAndResolve( // after it is initialized; `undefined` drives the worker no-store // fallback (return ParsedFiles in the result). workerPool = createWorkerPool(workerUrl, effectivePoolSize, { + ...(workerResources + ? { heapLimitMb: workerResources.workerHeapMb, requireDrain: true } + : {}), parsedFileStoreStoragePath: parsedFileStorePath, // Durable, content-addressed shard dir for warm-cache reuse (#2038). // Initialized below before the chunk loop (same deferred-init pattern @@ -817,6 +832,7 @@ export async function runChunkedParseAndResolve( let chunkCacheMisses = 0; let parseCacheHitFileCount = 0; let reparsedFileCount = 0; + let parseFailure: unknown; try { // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch @@ -1525,8 +1541,22 @@ export async function runChunkedParseAndResolve( `processRoutesFromExtracted: ${ms.toFixed(0)}ms (${allExtractedRoutes.length} routes)`, ); } + } catch (error) { + parseFailure = error; + throw error; } finally { - await workerPool?.terminate(); + try { + await workerPool?.terminate(); + } catch (drainError) { + if (parseFailure !== undefined) { + throw new AggregateError( + [parseFailure, drainError], + 'Parsing failed and native workers have not drained', + { cause: parseFailure }, + ); + } + throw drainError; + } } // Fetch calls + ORM queries were already extracted inside each worker diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 9ec3c5981..64b7a4522 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -255,6 +255,8 @@ export interface PipelineOptions { * analyze invocations. */ workerPoolSize?: number; + /** Admitted ceiling, separate from an explicit worker count so tiny repos retain auto-sizing. */ + workerResources?: Readonly<{ workers: number; workerHeapMb: number }>; /** * Process-detection budget (#3313). Explicit `maxProcesses` replaces the * dynamic `symbols / 10` formula; the other three replace compiled defaults. diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index b972d7326..d748815a3 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -220,6 +220,10 @@ export interface WorkerPoolStats { } export interface WorkerPoolOptions { + /** Per-run old-generation allocation; overrides the process-wide environment default. */ + heapLimitMb?: number; + /** Refuse to release the parser reservation while a retired native worker still runs. */ + requireDrain?: boolean; subBatchSize?: number; subBatchMaxBytes?: number; subBatchIdleTimeoutMs?: number; @@ -1053,6 +1057,7 @@ export const createWorkerPool = ( const size = poolSize ?? resolveAutoPoolSize(); const poolOptions = resolveWorkerPoolOptions(options, size); + const requireDrain = options?.requireDrain === true; // Production factory spawns with `{ stderr: true }` so a worker's crash // output is redirected to a `worker.stderr` stream we can tee + capture // (see captureWorkerStderr) and attach to readiness-failure messages — @@ -1071,7 +1076,12 @@ export const createWorkerPool = ( parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg ? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines } : undefined; - const workerHeapCapMb = resolveWorkerHeapCapMb(size); + const workerHeapCapMb = options?.heapLimitMb ?? resolveWorkerHeapCapMb(size); + if ( + options?.heapLimitMb !== undefined && + (!Number.isSafeInteger(workerHeapCapMb) || workerHeapCapMb < 512) + ) + throw new Error('Parser worker heap allocation must be a safe integer of at least 512 MiB'); // The 512MB per-worker floor exists so a worker can parse anything real, // but on a very small container a large pool of floored workers can still // overcommit total memory (#2649 review). Behavior is unchanged — deaths @@ -1135,6 +1145,7 @@ export const createWorkerPool = ( safePoint: Promise; }; const retiredWorkers = new Set(); + const pendingWorkerReleases = new Set>(); const respawnCount: number[] = new Array(size).fill(0); const activeSlots: Set = new Set(); // Layer 3 (quarantine): tracked via the dedicated `quarantine.ts` @@ -1492,7 +1503,7 @@ export const createWorkerPool = ( reason: string, ): void => { let cleaned = false; - let terminateStarted = false; + let termination: Promise | undefined; let resolveSafePoint!: () => void; const safePoint = new Promise((resolve) => { resolveSafePoint = resolve; @@ -1517,11 +1528,14 @@ export const createWorkerPool = ( retiredWorkers.delete(record); } - async function terminateRetired() { - if (terminateStarted) return; - terminateStarted = true; - cleanupRetired(); - await worker.terminate().catch(() => undefined); + function terminateRetired(): Promise { + termination ??= worker.terminate().then( + () => cleanupRetired(), + () => { + if (!requireDrain) cleanupRetired(); + }, + ); + return termination; } function terminateWhenBackInJs() { @@ -1582,9 +1596,14 @@ export const createWorkerPool = ( // Recovery must settle before dispatch returns, but a failed thread // may never acknowledge termination. Bound that wait as in shutdown. const termination = existing.terminate().then( - () => undefined, + () => { + pendingWorkerReleases.delete(termination); + }, () => undefined, ); + if (requireDrain) { + pendingWorkerReleases.add(termination); + } if (!(await settledWithin(termination, poolOptions.shutdownDrainMs))) { existing.unref?.(); logger.warn( @@ -1601,6 +1620,22 @@ export const createWorkerPool = ( ): Promise => { await removeWorkerFromSlot(workerIndex, mode, reason); if (stopped) return false; + if (requireDrain) { + // A timeout retires a native-active worker without killing it. Its + // slot remains reserved until both the safe point and native exit. + for (const record of retiredWorkers) { + if (record.workerIndex !== workerIndex) continue; + if ( + !record.safeToTerminate && + !(await settledWithin(record.safePoint, poolOptions.shutdownDrainMs)) + ) + return false; + if (!(await settledWithin(record.terminate(), poolOptions.shutdownDrainMs))) + return false; + if (retiredWorkers.has(record)) return false; + } + if (pendingWorkerReleases.size > 0 || stopped) return false; + } const replacement = spawnAndCapture(workerUrl); try { await waitForWorkerReady(replacement, poolOptions.workerReadyTimeoutMs); @@ -2418,6 +2453,10 @@ export const createWorkerPool = ( await terminateTrackedWorkers(workers); workers.length = 0; activeSlots.clear(); + if (requireDrain && (retiredWorkers.size > 0 || pendingWorkerReleases.size > 0)) + throw new Error( + 'Parser workers have not released their native resource reservation; refusing to start another analysis phase. Workers will exit at their next safe point.', + ); }; const dispatch = async ( diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 4aea794b1..83f283f3d 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -624,6 +624,7 @@ const writableFtsCrashWalEvidence = async ( const reopenWritableAfterMissingShadow = async ( dbPath: string, err: unknown, + options: LbugInitOptions = {}, ): Promise => { // Analyze writers may park a leftover in-place FTS abort WAL. Serve/embed // refuse first via assertReadOnlyFtsCrashSafe and must not pass this @@ -644,7 +645,12 @@ const reopenWritableAfterMissingShadow = async ( throw new Error(renameFailureMessage(dbPath, renameErr)); } - return await openLbugConnection(lbug, dbPath); + return options.bufferPoolBytes !== undefined || options.threads !== undefined + ? await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }) + : await openLbugConnection(lbug, dbPath); }; const ensureReadOnlyConnectionUsable = async ( @@ -802,7 +808,13 @@ const runSchemaCreationQueries = async (dbPath: string): Promise return null; }; -export const initLbug = async (dbPath: string, options: { skipFts?: boolean } = {}) => { +export interface LbugInitOptions { + skipFts?: boolean; + bufferPoolBytes?: number; + threads?: number; +} + +export const initLbug = async (dbPath: string, options: LbugInitOptions = {}) => { return runWithSessionLock(() => ensureLbugInitialized(dbPath, options)); }; @@ -856,34 +868,51 @@ export const withLbugDb = async ( }; let currentDbSkipFts = false; +let currentDbBufferPoolBytes: number | undefined; +let currentDbThreads: number | undefined; +let nativeCloseUnverified = false; const ensureLbugInitialized = async ( dbPath: string, - options: { readOnly?: boolean; skipFts?: boolean } = {}, + options: LbugInitOptions & { readOnly?: boolean } = {}, ) => { const readOnly = options.readOnly === true; const skipFts = options.skipFts === true; + if ( + options.bufferPoolBytes !== undefined && + (!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes < 8 * 1024 * 1024) + ) + throw new Error('Explicit LadybugDB buffer pool must be at least 8 MiB'); + if ( + options.threads !== undefined && + (!Number.isSafeInteger(options.threads) || options.threads < 1) + ) + throw new Error('Explicit LadybugDB thread allocation must be a positive integer'); if ( conn && + !nativeCloseUnverified && currentDbPath === dbPath && currentDbReadOnly === readOnly && - currentDbSkipFts === skipFts + currentDbSkipFts === skipFts && + currentDbBufferPoolBytes === options.bufferPoolBytes && + currentDbThreads === options.threads ) { return { db, conn }; } - await doInitLbug(dbPath, { readOnly, skipFts }); + await doInitLbug(dbPath, { ...options, readOnly, skipFts }); return { db, conn }; }; const doInitLbug = async ( dbPath: string, - options: { readOnly?: boolean; skipFts?: boolean } = {}, + options: LbugInitOptions & { readOnly?: boolean } = {}, ) => { const readOnly = options.readOnly === true; const skipFts = options.skipFts === true; // Different database requested — close the old one first - if (conn || db) { - await safeClose(); + const resourceBound = options.bufferPoolBytes !== undefined || options.threads !== undefined; + if (conn || db || nativeCloseUnverified || (resourceBound && currentDbPath)) { + await safeClose({ strict: resourceBound }); currentDbPath = null; ftsLoaded = false; vectorExtensionLoaded = false; @@ -1045,10 +1074,19 @@ const doInitLbug = async ( }); try { - const opened = await openLbugConnection(lbug, dbPath); + const opened = resourceBound + ? await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }) + : await openLbugConnection(lbug, dbPath); db = opened.db; conn = opened.conn; currentDbReadOnly = false; + // Record the live allocation before schema work, so an initialization + // failure must also prove release before another native open. + currentDbBufferPoolBytes = options.bufferPoolBytes; + currentDbThreads = options.threads; } catch (err) { // Incremental analyze can hit a storage-version mismatch on construct // or (more often) on the first schema query below. Fail immediately @@ -1066,7 +1104,7 @@ const doInitLbug = async ( if (missingShadowError) { await safeClose(); resetOpenConnectionState(); - const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError); + const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError, options); db = reopened.db; conn = reopened.conn; currentDbReadOnly = false; @@ -1094,6 +1132,8 @@ const doInitLbug = async ( } currentDbSkipFts = skipFts; + currentDbBufferPoolBytes = options.bufferPoolBytes; + currentDbThreads = options.threads; currentDbPath = dbPath; return { db, conn }; }; @@ -2449,7 +2489,13 @@ export const tryFlushWAL = async (): Promise => { * @see flushWAL — CHECKPOINT-only (connection stays open) * @see closeLbug — safeClose + module state reset (full teardown) */ -export const safeClose = async (): Promise => { +export const safeClose = async (options: { strict?: boolean } = {}): Promise => { + const strict = + options.strict === true || + nativeCloseUnverified || + currentDbBufferPoolBytes !== undefined || + currentDbThreads !== undefined; + if (strict) nativeCloseUnverified = true; await flushWAL(); // Capture before close — currentDbPath stays set so the Windows post-close // probe below knows which file to wait on. @@ -2458,7 +2504,8 @@ export const safeClose = async (): Promise => { try { // eslint-disable-next-line no-restricted-syntax -- sole authorised close site await conn.close(); - } catch { + } catch (error) { + if (strict) throw error; /* best-effort */ } conn = null; @@ -2467,7 +2514,8 @@ export const safeClose = async (): Promise => { try { // eslint-disable-next-line no-restricted-syntax -- sole authorised close site await db.close(); - } catch { + } catch (error) { + if (strict) throw error; /* best-effort */ } db = null; @@ -2480,6 +2528,7 @@ export const safeClose = async (): Promise => { if (process.platform === 'win32' && closingDbPath) { const released = await waitForWindowsHandleRelease(closingDbPath); if (!released) { + if (strict) throw new Error('LadybugDB native resource reservation has not been released'); // Probe exhausted with a lock code still in flight. The next // openLbugConnection will absorb whatever residual lag remains, but // a chronic warning helps operators spot AV interference (Windows @@ -2493,6 +2542,7 @@ export const safeClose = async (): Promise => { if (closingDbPath) { await finalizeLbugSidecarsAfterClose(closingDbPath, { logger }); } + nativeCloseUnverified = false; }; /** @@ -2528,9 +2578,11 @@ export const closeLbugBeforeExit = async (): Promise => { // double-free. }; -export const closeLbug = async (): Promise => { - await safeClose(); +export const closeLbug = async (options: { strict?: boolean } = {}): Promise => { + await safeClose(options); currentDbPath = null; + currentDbBufferPoolBytes = undefined; + currentDbThreads = undefined; ftsLoaded = false; vectorExtensionLoaded = false; vectorIndexEnsured = false; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3d6d53a5c..9397374fc 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -462,6 +462,8 @@ export interface AnalyzeCallbacks { } export interface AnalyzeOptions { + /** Frozen per-invocation allocations; entrypoints resolve these before destructive work. */ + resources?: import('./analysis-resource-budget.js').AnalysisResourceBudget; /** * Rebuild the graph and FTS. Parser output is still reused from the * content-addressed parse cache unless `useParseCache` is false. @@ -1648,7 +1650,15 @@ async function runFullAnalysisInner( ): Promise { const ftsDisabledReason = resolveFtsDisableReason(options.skipFts); const initAnalysisLbug = (dbPath: string) => - ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath); + options.resources + ? initLbug(dbPath, { + skipFts: !!ftsDisabledReason, + bufferPoolBytes: options.resources.parse.graphPoolBytes, + threads: options.resources.parse.nativeThreads, + }) + : ftsDisabledReason + ? initLbug(dbPath, { skipFts: true }) + : initLbug(dbPath); const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); @@ -3176,6 +3186,7 @@ async function runFullAnalysisInner( parsedFileStorePath: coldParseRebuildDir, parseWorkerUrl: parseIdentityResolution.ok ? parseIdentityResolution.workerUrl : undefined, workerPoolSize: options.workerPoolSize, + ...(options.resources ? { workerResources: options.resources.parse } : {}), maxProcesses: processDetectionBudget.maxProcesses, maxProcessBranching: processDetectionBudget.overridden.maxProcessBranching ? processDetectionBudget.maxProcessBranching diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 238d0fcc3..8a5121287 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -49,6 +49,16 @@ describe('analysis resource admission', () => { expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(1024 * MIB); }); + it('preserves the heap-only flag minimum without silently raising it', () => { + const budget = resolveAnalysisResourceBudget({ ...request, mainHeapMb: 200 }); + expect(budget.mainHeapMb).toBe(200); + expect(budget.parse.nativeThreads).toBe(request.availableCpus); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => resolveAnalysisResourceBudget({ ...request, mainHeapMb: 199 })).toThrow( + 'cannot admit', + ); + }); + it('admits the full working set of a supported 10 MiB node with a larger budget', () => { const budget = resolveAnalysisResourceBudget({ ...request, diff --git a/gitnexus/test/unit/lbug-analysis-resources.test.ts b/gitnexus/test/unit/lbug-analysis-resources.test.ts new file mode 100644 index 000000000..d9f666ca1 --- /dev/null +++ b/gitnexus/test/unit/lbug-analysis-resources.test.ts @@ -0,0 +1,126 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const native = vi.hoisted(() => ({ + open: vi.fn(), + close: vi.fn(async () => {}), + quarantine: vi.fn(async () => ({ quarantinedPath: '/unused' })), +})); +vi.mock('../../src/core/lbug/lbug-config.js', async (original) => ({ + ...(await original()), + openLbugConnection: native.open, + closeLbugConnection: native.close, +})); +vi.mock('../../src/core/lbug/sidecar-recovery.js', async (original) => ({ + ...(await original()), + quarantineWalForMissingShadow: native.quarantine, + isMissingShadowSidecarError: (error: unknown) => + error instanceof Error && error.message === 'missing shadow test', +})); +vi.mock('../../src/core/lbug/extension-loader.js', async (original) => ({ + ...(await original()), + extensionManager: { + ensure: vi.fn(async () => true), + reset: vi.fn(), + getCapabilities: vi.fn(() => []), + }, +})); + +describe('analysis database resource allocations', () => { + let root: string; + let adapter: typeof import('../../src/core/lbug/lbug-adapter.js'); + beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-analysis-resources-')); + native.open.mockImplementation(async () => ({ + db: { close: vi.fn(async () => {}) }, + conn: { + query: vi.fn(async () => ({ getAll: async () => [], close: () => {} })), + close: vi.fn(async () => {}), + }, + })); + adapter = await import('../../src/core/lbug/lbug-adapter.js'); + }); + afterEach(async () => { + await adapter.closeLbug(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('applies native allocations and reopens only when the allocation changes', async () => { + const dbPath = path.join(root, 'lbug'); + const options = { skipFts: true, bufferPoolBytes: 64 * 1024 * 1024, threads: 3 }; + await adapter.initLbug(dbPath, options); + await adapter.initLbug(dbPath, { ...options }); + expect(native.open).toHaveBeenCalledTimes(1); + expect(native.open.mock.calls[0][2]).toEqual({ + bufferPoolBytes: options.bufferPoolBytes, + threads: 3, + }); + const first = await native.open.mock.results[0].value; + await adapter.initLbug(dbPath, { ...options, threads: 2 }); + expect(first.conn.close).toHaveBeenCalledTimes(1); + expect(first.db.close).toHaveBeenCalledTimes(1); + expect(native.open).toHaveBeenCalledTimes(2); + expect(native.open.mock.calls[1][2]).toEqual({ + bufferPoolBytes: options.bufferPoolBytes, + threads: 2, + }); + }); + + it('rejects invalid allocations before creating the database directory or opening native code', async () => { + const parent = path.join(root, 'not-created'); + for (const options of [ + { bufferPoolBytes: 0 }, + { bufferPoolBytes: 1024 }, + { threads: 0 }, + { threads: 1.5 }, + ]) { + await expect(adapter.initLbug(path.join(parent, 'lbug'), options)).rejects.toThrow( + /Explicit LadybugDB/, + ); + } + expect(native.open).not.toHaveBeenCalled(); + await expect(fs.stat(parent)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.each(['conn', 'db'] as const)( + 'retains a failed %s close and refuses to allocate another pool', + async (handle) => { + const dbPath = path.join(root, 'lbug'); + const options = { skipFts: true, bufferPoolBytes: 64 * 1024 * 1024, threads: 3 }; + await adapter.initLbug(dbPath, options); + const first = await native.open.mock.results[0].value; + first[handle].close.mockRejectedValueOnce(new Error('native close failed')); + await expect(adapter.initLbug(dbPath, { ...options, threads: 2 })).rejects.toThrow( + 'native close failed', + ); + expect(native.open).toHaveBeenCalledTimes(1); + // A subsequent successful drain can safely admit the requested allocation. + await adapter.initLbug(dbPath, options); + expect(native.open).toHaveBeenCalledTimes(2); + }, + ); + + it('keeps the same allocations when missing-shadow recovery reopens the writer', async () => { + native.open.mockResolvedValueOnce({ + db: { close: vi.fn(async () => {}) }, + conn: { + query: vi.fn(async () => { + throw new Error('missing shadow test'); + }), + close: vi.fn(async () => {}), + }, + }); + const options = { skipFts: true, bufferPoolBytes: 32 * 1024 * 1024, threads: 2 }; + await adapter.initLbug(path.join(root, 'lbug'), options); + expect(native.quarantine).toHaveBeenCalledTimes(1); + expect(native.open).toHaveBeenCalledTimes(2); + expect(native.open.mock.calls.map((call) => call[2])).toEqual([ + { bufferPoolBytes: options.bufferPoolBytes, threads: 2 }, + { bufferPoolBytes: options.bufferPoolBytes, threads: 2 }, + ]); + }); +}); diff --git a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts index d9bd574bb..ce7423d6d 100644 --- a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts +++ b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts @@ -341,6 +341,70 @@ describe('parse-impl worker pool lazy startup', () => { } }); + it('keeps tiny-repository automatic sizing inside the admitted worker ceiling', async () => { + const saved = process.env.GITNEXUS_WORKER_POOL_SIZE; + delete process.env.GITNEXUS_WORKER_POOL_SIZE; + try { + const rels = ['a.ts', 'b.ts', 'c.ts', 'd.ts']; + const scanned = rels.map((rel) => { + fs.writeFileSync(path.join(repoDir, rel), 'export const value = 1;'); + return { path: rel, size: 23 }; + }); + const markerDir = path.join(tempDir, 'budget-markers'); + const workerPath = path.join(tempDir, 'budget-worker.cjs'); + // Real worker reports V8's assigned old-generation limit. The generous + // default is several GiB on this host; this run must stay at 512 MiB. + fs.writeFileSync( + workerPath, + ` +const fs = require('node:fs'); +const { parentPort, resourceLimits, threadId } = require('node:worker_threads'); +fs.mkdirSync(${JSON.stringify(markerDir)}, { recursive: true }); +fs.writeFileSync(${JSON.stringify(markerDir)} + '/' + threadId, String(resourceLimits.maxOldGenerationSizeMb)); +parentPort.postMessage({ type: 'ready' }); +parentPort.on('message', (msg) => { + if (msg.type === 'sub-batch') parentPort.postMessage({ type: 'sub-batch-done' }); + if (msg.type === 'flush') parentPort.postMessage({ type: 'result', data: ${JSON.stringify(emptyWorkerResult('a.ts', 'value'))} }); +}); +`, + ); + await runChunkedParseAndResolve( + createKnowledgeGraph(), + scanned, + rels, + 4, + repoDir, + Date.now(), + () => {}, + { + workerUrlForTest: pathToFileURL(workerPath), + workerResources: { workers: 3, workerHeapMb: 512 }, + }, + ); + const markers = fs.readdirSync(markerDir); + expect(markers).toHaveLength(1); + expect(Number(fs.readFileSync(path.join(markerDir, markers[0]), 'utf8'))).toBe(512); + await expect( + runChunkedParseAndResolve( + createKnowledgeGraph(), + scanned, + rels, + 4, + repoDir, + Date.now(), + () => {}, + { + workerPoolSize: 4, + workerResources: { workers: 3, workerHeapMb: 512 }, + }, + ), + ).rejects.toThrow('admitted resource budget'); + } finally { + if (saved === undefined) delete process.env.GITNEXUS_WORKER_POOL_SIZE; + else process.env.GITNEXUS_WORKER_POOL_SIZE = saved; + } + }); + it('throws when GITNEXUS_WORKER_POOL_SIZE=0 and no --workers flag (sequential parsing removed)', async () => { const saved = process.env.GITNEXUS_WORKER_POOL_SIZE; process.env.GITNEXUS_WORKER_POOL_SIZE = '0'; diff --git a/gitnexus/test/unit/worker-pool-resource-limits.test.ts b/gitnexus/test/unit/worker-pool-resource-limits.test.ts index 94ea73346..6a8fdfc20 100644 --- a/gitnexus/test/unit/worker-pool-resource-limits.test.ts +++ b/gitnexus/test/unit/worker-pool-resource-limits.test.ts @@ -163,4 +163,35 @@ describe('resolveWorkerHeapCapMb (#2649 per-worker heap cap)', () => { fs.rmSync(tempDir, { recursive: true, force: true }); } }); + + it('uses a per-run heap allocation without changing later pools or environment overrides', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-worker-run-limits-')); + const workerPath = path.join(tempDir, 'fake-worker.js'); + fs.writeFileSync(workerPath, '// fake worker'); + process.env.GITNEXUS_WORKER_HEAP_MB = '256'; + try { + const { createWorkerPool } = await import('../../src/core/ingestion/workers/worker-pool.js'); + const bounded = createWorkerPool(pathToFileURL(workerPath), 1, { heapLimitMb: 768 }); + await bounded.terminate(); + const legacy = createWorkerPool(pathToFileURL(workerPath), 1); + await legacy.terminate(); + expect(workerCtorOptions).toEqual([ + expect.objectContaining({ + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: 768 }, + }), + expect.objectContaining({ + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: 256 }, + }), + ]); + expect(process.env.GITNEXUS_WORKER_HEAP_MB).toBe('256'); + for (const heapLimitMb of [0, 511, 512.5, NaN, Infinity]) { + expect(() => createWorkerPool(pathToFileURL(workerPath), 1, { heapLimitMb })).toThrow( + 'heap allocation', + ); + } + expect(workerCtorOptions).toHaveLength(2); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index 29da7f18f..bfa170b18 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -94,6 +94,71 @@ afterEach(() => { }); describe('worker pool timeout retirement', () => { + it('refuses replacement and phase handoff while a budgeted native worker has not drained', async () => { + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + heapLimitMb: 512, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 20, + workerFactory: () => + new TimeoutThenHealthyWorker() as unknown as import('node:worker_threads').Worker, + }); + try { + await expect( + pool.dispatch([{ path: 'src/stalled.ts', content: 'const x = 1;' }]), + ).rejects.toThrow(); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + expect(TimeoutThenHealthyWorker.instances[0].terminateCalls).toBe(0); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(TimeoutThenHealthyWorker.instances[0].terminateCalls).toBe(0); + } finally { + TimeoutThenHealthyWorker.instances[0].emit('message', { type: 'sub-batch-done' }); + await waitFor( + () => TimeoutThenHealthyWorker.instances[0].terminateCalls === 1, + 'Worker did not retire', + ); + await pool.terminate(); + } + }); + + it('replaces a budgeted worker only after its safe point and native termination complete', async () => { + TimeoutThenHealthyWorker.firstWorkerBehavior = 'delayed-safe-return'; + TimeoutThenHealthyWorker.safeReturnDelayMs = 40; + let completeTermination: (() => void) | undefined; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 500, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id === 0) + worker.terminate = () => { + worker.terminateCalls++; + return new Promise((resolve) => { + completeTermination = () => { + worker.emit('exit', 0); + resolve(0); + }; + }); + }; + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + const result = pool.dispatch([{ path: 'src/delayed.ts', content: '' }]); + await waitFor(() => completeTermination !== undefined, 'Missing safe termination'); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + completeTermination!(); + await expect(result).resolves.toEqual([{ paths: ['src/delayed.ts'] }]); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(2); + } finally { + completeTermination?.(); + await pool.terminate(); + } + }); + it('does not immediately terminate a worker that timed out inside native parsing', async () => { const pool = createWorkerPool(workerUrl, 1, { subBatchIdleTimeoutMs: 20, From 1c95ebe00a468263bc549c18bd8c303018946598 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 15:26:05 +0000 Subject: [PATCH 09/67] feat(embeddings): reserve sidecar resources Pin each durable job to an admitted model heap and CPU config. Retain native allocations after caller cancellation until response or process exit, and reject phase handoff when release is unverified. --- gitnexus/src/core/analysis-resource-budget.ts | 17 +- gitnexus/src/core/embeddings/embedding-job.ts | 57 +++++-- .../embeddings/embedding-sidecar-client.ts | 159 ++++++++++++++++-- .../embedding-job-controller.test.ts | 78 +++++++++ .../unit/analysis-resource-budget.test.ts | 16 ++ .../unit/embedding-sidecar-client.test.ts | 127 ++++++++++++++ 6 files changed, 422 insertions(+), 32 deletions(-) diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 5cd86c508..95c81dd61 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -15,6 +15,7 @@ export interface AnalysisResourceRequest { mainHeapMb?: number; workers?: number; workerHeapMb?: number; + embeddingThreads?: number; /** Existing automatic worker policy; explicit workers still wins. */ preferredWorkers?: number; /** Resume/sync can admit embeddings without reserving parser workers. */ @@ -42,6 +43,8 @@ export interface AnalysisResourceBudget { readonly graphPoolBytes: number; readonly vectorPoolBytes: number; readonly modelBytes: number; + /** Sidecar old-generation allocation, included in modelBytes with native/young space. */ + readonly modelHeapMb: number; /** Combined source, expanded text, encoded request, vectors and prefetch. */ readonly queueBytes: number; readonly nativeThreads: number; @@ -66,6 +69,11 @@ export function resolveAnalysisResourceBudget( ): AnalysisResourceBudget { const memory = integer(request.effectiveMemoryBytes, 'effectiveMemoryBytes'); const cpus = integer(request.availableCpus, 'availableCpus'); + const embeddingThreads = + request.embeddingThreads === undefined + ? Math.min(4, cpus) + : integer(request.embeddingThreads, 'embedding threads'); + if (embeddingThreads > cpus) throw new Error('Embedding threads exceed the CPU allowance'); const availableMb = Math.floor(memory / MIB); const totalMb = request.totalMb === undefined @@ -81,10 +89,14 @@ export function resolveAnalysisResourceBudget( const graphPoolBytes = request.graphPoolBytes === undefined ? bytes(Math.max(GRAPH_POOL_MB, Math.min(4096, Math.floor(totalMb * 0.15))), 'graph pool') - : integer(request.graphPoolBytes, 'graph pool bytes'); + : integer(request.graphPoolBytes, 'graph pool bytes', 8 * MIB); // Zero is the native engine's unbounded override, never a bounded allocation. const graphPoolMb = Math.ceil(graphPoolBytes / MIB); const modelMb = integer(request.modelMemoryMb, 'model reservation', 0); + if (modelMb > 0 && modelMb < 512) + throw new Error( + 'Local model reservation must be at least 512 MiB for heap and native allocations', + ); // A supported 10 MiB source node needs its full text/AST working set. Larger // explicit budgets must be able to admit it without truncating its content. const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32))); @@ -172,8 +184,9 @@ export function resolveAnalysisResourceBudget( graphPoolBytes, vectorPoolBytes: bytes(VECTOR_POOL_MB, 'vector pool'), modelBytes: bytes(modelMb, 'model reservation'), + modelHeapMb: modelMb === 0 ? 0 : Math.min(1024, Math.floor(modelMb / 4)), queueBytes: bytes(queueMb, 'embedding queues'), - nativeThreads: Math.min(4, cpus), + nativeThreads: embeddingThreads, reservedBytes: bytes(embeddingReservedMb, 'embedding reservation'), }), }); diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index ed4bc9fd5..aec5655a8 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -29,6 +29,7 @@ import { runEmbeddingJob } from './embedding-job-runner.js'; import { publishVectorGeneration, type VectorPublicationActions } from './vector-publication.js'; import type { EmbeddingConfig, ModelProgressCallback } from './types.js'; import type { embedBatch } from './embedder.js'; +import type { EmbeddingSidecarReservation } from './embedding-sidecar-client.js'; type GraphSource = EmbeddingManifestSource & { close(): Promise }; export interface PublishedEmbeddingJobOptions { @@ -149,6 +150,15 @@ export async function runPublishedEmbeddingJob( throw new Error('Invalid embedding resource reservation'); if (!Number.isSafeInteger(resources.nativeThreads) || resources.nativeThreads < 1) throw new Error('Invalid embedding CPU reservation'); + const usesLocalRuntime = vector.provider === 'local' && !dependencies.provider; + if ( + usesLocalRuntime && + (!Number.isSafeInteger(resources.modelBytes) || + !Number.isSafeInteger(resources.modelHeapMb) || + resources.modelHeapMb < 128 || + resources.modelHeapMb * 1024 * 1024 + 48 * 1024 * 1024 >= resources.modelBytes) + ) + throw new Error('Local embedding runtime does not fit its resource reservation'); const identities = embeddingJobIdentity( { generation: graph.id, @@ -193,6 +203,21 @@ export async function runPublishedEmbeddingJob( } else if (previous && previous.inputIdentity !== identities.inputIdentity) { options.log?.('Embedding text options changed; prior vectors are not compatible.'); } + const reusable = + previous && + previous.identity !== identities.identity && + previous.vectorIdentity === identities.vectorIdentity && + previous.inputIdentity === identities.inputIdentity && + previous.dimensions === vector.dimensions; + // A reuse reader shares the vector reservation; it is not a third default pool. + const poolBytes = Math.floor(resources.vectorPoolBytes / (reusable ? 2 : 1)); + const storeResources = { + bufferPoolBytes: poolBytes, + threads: resources.nativeThreads, + batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), + }; + if (poolBytes < 8 * 1024 * 1024 || storeResources.batchBytes < vector.dimensions * 16) + throw new Error('Vector writer and reuse reader do not fit their resource reservation'); const directory = await jobDirectory(root, identities.identity); if (metadata.embeddingJob?.identity !== identities.identity) { metadata = { @@ -209,20 +234,15 @@ export async function runPublishedEmbeddingJob( let source: GraphSource | undefined; let store: VectorStore | undefined; let reuse: VectorStore | undefined; - const reusable = - previous && - previous.identity !== identities.identity && - previous.vectorIdentity === identities.vectorIdentity && - previous.inputIdentity === identities.inputIdentity && - previous.dimensions === vector.dimensions; - // A reuse reader shares the vector reservation; it is not a third default pool. - const poolBytes = Math.floor(resources.vectorPoolBytes / (reusable ? 2 : 1)); - const storeResources = { - bufferPoolBytes: poolBytes, - threads: resources.nativeThreads, - batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), - }; + let runtime: EmbeddingSidecarReservation | undefined; try { + if (usesLocalRuntime) { + const { reserveEmbeddingSidecar } = await import('./embedding-sidecar-client.js'); + runtime = await reserveEmbeddingSidecar(config, resources.modelHeapMb); + } else if (!dependencies.provider) { + const { drainEmbeddingSidecarForAnalysis } = await import('./embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + } const manifestExists = await existsOwnedFile(path.join(directory, 'manifest.json')); if (!manifestExists) await clearUnstartedManifest(directory); const getSource = async (): Promise => @@ -258,6 +278,7 @@ export async function runPublishedEmbeddingJob( const provider: typeof embedBatch = dependencies.provider ?? (async (texts, request) => { + if (runtime) return runtime.embedBatch(texts, request); const { embedBatch } = await import('./embedder.js'); return embedBatch(texts, { ...request, @@ -266,9 +287,7 @@ export async function runPublishedEmbeddingJob( }); const initializeProvider = async () => { if (dependencies.initializeProvider) return dependencies.initializeProvider(config); - if (vector.provider !== 'local') return; - const { initEmbedder } = await import('./embedder.js'); - await initEmbedder(options.onModelProgress, config); + await runtime?.initialize(options.onModelProgress); }; const result = await runEmbeddingJob({ directory, @@ -334,7 +353,11 @@ export async function runPublishedEmbeddingJob( try { await reuse?.close(); } finally { - await source?.close(); + try { + await source?.close(); + } finally { + runtime?.release(); + } } } } diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts index ce8e1e2f0..690c03559 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts @@ -24,6 +24,8 @@ import type { SidecarResponse, } from './embedding-sidecar-protocol.js'; import { logger } from '../logger.js'; +import { isDeepStrictEqual } from 'node:util'; +import { resolveEmbeddingConfig } from './config.js'; export type ForkImpl = ( modulePath: string, @@ -47,10 +49,16 @@ let lastInitOptions: | { embeddingConfig?: Partial; forceDevice?: EmbeddingSidecarDevice; + heapLimitMb?: number; } | undefined; let exitHooked = false; let progressSink: ((progress: ModelProgress) => void) | undefined; +// A caller abort only stops waiting. The child may still be inside ONNX; keep +// its physical requests and process allocation until a response/close proves release. +const physicalRequests = new Map(); +const liveChildren = new Set(); +let runtimeReservation: { config: EmbeddingConfig; heapLimitMb: number } | undefined; const pending = new Map< number, @@ -75,6 +83,9 @@ export const _resetEmbeddingSidecarForTests = (): void => { device = 'cpu'; ensureChain = null; lastInitOptions = undefined; + runtimeReservation = undefined; + physicalRequests.clear(); + liveChildren.clear(); }; const sidecarScriptPath = (): string => { @@ -91,9 +102,16 @@ const tsxHookArgs = (): string[] => { return ['--import', pathToFileURL(require.resolve('tsx/esm')).href]; }; -const childEnv = (): NodeJS.ProcessEnv => { +const childEnv = (configPinned = false): NodeJS.ProcessEnv => { const env = { ...process.env }; delete env.GITNEXUS_EMBEDDING_URL; + if (configPinned) { + // The admitted, resolved config travels over IPC. An environment mutation + // between admission and lazy model init must not override its allocations. + for (const name of ['THREADS', 'DEVICE', 'BATCH_SIZE', 'SUB_BATCH_SIZE', 'MAX_INPUT_CHARS']) { + delete env[`GITNEXUS_EMBEDDING_${name}`]; + } + } return env; }; @@ -146,10 +164,14 @@ const rejectAll = (error: Error): void => { }; const attachChild = (proc: ChildProcess): void => { + liveChildren.add(proc); proc.stderr?.on('data', (chunk: Buffer | string) => { logger.debug({ sidecar: true }, String(chunk).trimEnd()); }); proc.on('message', (msg: SidecarResponse) => { + if (msg.type !== 'progress' && physicalRequests.get(msg.id) === proc) + physicalRequests.delete(msg.id); + if (child !== proc) return; if (msg.type === 'progress') { progressSink?.({ status: msg.status, @@ -162,6 +184,8 @@ const attachChild = (proc: ChildProcess): void => { waiter.resolve(msg); }); proc.on('close', (code, signal) => { + liveChildren.delete(proc); + for (const [id, owner] of physicalRequests) if (owner === proc) physicalRequests.delete(id); if (child !== proc) return; noteChildDeath(signal); child = null; @@ -183,6 +207,7 @@ const request = ( signal?: AbortSignal, ): Promise => { if (!child) return Promise.reject(new Error('Embedding sidecar is not running')); + const proc = child; try { signal?.throwIfAborted(); } catch (err) { @@ -210,7 +235,7 @@ const request = ( }; const timer = setTimeout(() => { finish(() => { - child?.kill('SIGKILL'); + proc.kill('SIGKILL'); reject(new Error(`Embedding sidecar request timed out after ${timeoutMs}ms (${msg.type})`)); }); }, timeoutMs); @@ -225,8 +250,10 @@ const request = ( return; } try { - child!.send({ ...msg, id } as SidecarRequest); + physicalRequests.set(id, proc); + proc.send({ ...msg, id } as SidecarRequest); } catch (err) { + physicalRequests.delete(id); finish(() => { reject(err instanceof Error ? err : new Error(String(err))); }); @@ -234,11 +261,19 @@ const request = ( }); }; -const spawnSidecar = (): ChildProcess => { +const spawnSidecar = (heapLimitMb?: number): ChildProcess => { + if (runtimeReservation && liveChildren.size > 0) { + throw new Error('Previous embedding sidecar has not released its native resource reservation'); + } const proc = forkImpl(sidecarScriptPath(), [], { - execArgv: tsxHookArgs(), + execArgv: [ + ...tsxHookArgs(), + ...(heapLimitMb === undefined + ? [] + : [`--max-old-space-size=${heapLimitMb}`, '--max-semi-space-size=16']), + ], stdio: ['ignore', 'ignore', 'pipe', 'ipc'], - env: childEnv(), + env: childEnv(runtimeReservation !== undefined), }); if (!exitHooked) { exitHooked = true; @@ -263,21 +298,32 @@ export const reapEmbeddingSidecar = (): void => { }; /** Reap and wait for the killed child's close/error so an awaited dispose is a real boundary. */ -export const reapEmbeddingSidecarAndWait = async (timeoutMs = 5_000): Promise => { +export const reapEmbeddingSidecarAndWait = async ( + timeoutMs = 5_000, + options: { requireExit?: boolean } = {}, +): Promise => { const proc = child; if (!proc) return; const closed = new Promise((resolve) => { const finish = (): void => resolve(); proc.once('close', finish); - proc.once('error', finish); + if (!options.requireExit) proc.once('error', finish); }); reapEmbeddingSidecar(); let timer: ReturnType | undefined; try { await Promise.race([ closed, - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs); + new Promise((resolve, reject) => { + timer = setTimeout( + () => + options.requireExit + ? reject( + new Error('Embedding sidecar has not released its native resource reservation'), + ) + : resolve(), + timeoutMs, + ); }), ]); } finally { @@ -314,11 +360,15 @@ const spawnAndInit = async (options?: EnsureSidecarOptions): Promise => { } const persisted = { - embeddingConfig: options?.embeddingConfig ?? lastInitOptions?.embeddingConfig, - forceDevice: options?.forceDevice ?? lastInitOptions?.forceDevice, + embeddingConfig: + runtimeReservation?.config ?? + resolveEmbeddingConfig(options?.embeddingConfig ?? lastInitOptions?.embeddingConfig), + forceDevice: + options?.forceDevice ?? (runtimeReservation ? undefined : lastInitOptions?.forceDevice), + heapLimitMb: runtimeReservation?.heapLimitMb ?? lastInitOptions?.heapLimitMb, }; - child = spawnSidecar(); + child = spawnSidecar(persisted.heapLimitMb); attachChild(child); progressSink = options?.onProgress; try { @@ -386,6 +436,26 @@ const raceAbort = async (promise: Promise, signal?: AbortSignal): Promise< export const ensureEmbeddingSidecar = async ( options?: EnsureSidecarOptions, ): Promise<{ device: EmbeddingSidecarDevice }> => { + if (runtimeReservation) { + if ( + options?.forceDevice && + runtimeReservation.config.device !== 'auto' && + options.forceDevice !== runtimeReservation.config.device + ) { + throw new Error('Embedding device conflicts with an active job resource reservation'); + } + for (const [key, value] of Object.entries(options?.embeddingConfig ?? {})) { + if ( + value !== undefined && + value !== runtimeReservation.config[key as keyof EmbeddingConfig] + ) { + throw new Error( + 'Embedding configuration conflicts with an active job resource reservation', + ); + } + } + options = { ...options, embeddingConfig: runtimeReservation.config }; + } if (localUnavailable) throw localUnavailableError(); if (ready && child) { rejectConflictingForceDevice(options?.forceDevice); @@ -403,6 +473,69 @@ export const ensureEmbeddingSidecar = async ( return { device }; }; +export interface EmbeddingSidecarReservation { + initialize(onProgress?: (progress: ModelProgress) => void): Promise; + embedBatch(texts: string[], options?: { signal?: AbortSignal }): Promise; + release(): void; +} + +/** Admit before opening job databases; acquire does not load a model. A warm + * compatible sidecar remains usable, and only one job can reconfigure it. */ +export async function reserveEmbeddingSidecar( + config: EmbeddingConfig, + heapLimitMb: number, +): Promise { + if (!Number.isSafeInteger(heapLimitMb) || heapLimitMb < 128) + throw new Error('Embedding sidecar heap allocation must be at least 128 MiB'); + if (runtimeReservation || ensureChain || physicalRequests.size > 0) + throw new Error('Embedding runtime is busy; cannot reassign its resource reservation'); + const reservation = { config: Object.freeze({ ...config }), heapLimitMb }; + runtimeReservation = reservation; + try { + if ( + child && + (!ready || + lastInitOptions?.heapLimitMb !== heapLimitMb || + !isDeepStrictEqual(lastInitOptions?.embeddingConfig, config)) + ) { + await reapEmbeddingSidecarAndWait(5_000, { requireExit: true }); + } + if (liveChildren.size > (child ? 1 : 0)) + throw new Error( + 'Previous embedding sidecar has not released its native resource reservation', + ); + const assertOwned = () => { + if (runtimeReservation !== reservation) + throw new Error('Embedding runtime reservation was released'); + }; + return { + initialize: async (onProgress) => { + assertOwned(); + await ensureEmbeddingSidecar({ onProgress, embeddingConfig: reservation.config }); + }, + embedBatch: async (texts, options) => { + assertOwned(); + return sidecarEmbedBatch(texts, options); + }, + release: () => { + if (runtimeReservation === reservation) runtimeReservation = undefined; + }, + }; + } catch (error) { + if (runtimeReservation === reservation) runtimeReservation = undefined; + throw error; + } +} + +/** Parser phases reuse the model reservation only after an idle child exits. */ +export async function drainEmbeddingSidecarForAnalysis(): Promise { + if (runtimeReservation || ensureChain || physicalRequests.size > 0) + throw new Error('Embedding runtime is busy; refusing to reuse its native resource reservation'); + await reapEmbeddingSidecarAndWait(5_000, { requireExit: true }); + if (liveChildren.size > 0) + throw new Error('Previous embedding sidecar has not released its native resource reservation'); +} + const vectorsFromEmbedResponse = (response: SidecarResponse): Float32Array[] => { if (response.type === 'error') throw new Error(response.message); if (response.type !== 'vectors') { diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index 3067715b0..e3dcd688a 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -96,6 +96,7 @@ async function fixture(count = 130) { graphPoolBytes: 16 * 1024 * 1024, vectorPoolBytes: 64 * 1024 * 1024, modelBytes: 0, + modelHeapMb: 0, queueBytes: 16 * 1024 * 1024, nativeThreads: 1, reservedBytes: 96 * 1024 * 1024, @@ -134,6 +135,83 @@ async function fixture(count = 130) { } describe('published graph embedding job controller', () => { + it('drains an earlier local runtime before admitting an HTTP job with no model reservation', async () => { + const f = await fixture(3); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'https://embedding.test/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'test-model'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + const sidecar = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const drain = vi.spyOn(sidecar, 'drainEmbeddingSidecarForAnalysis').mockResolvedValue(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { input: string[] }; + return new Response( + JSON.stringify({ + data: body.input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })), + }), + ); + }); + const result = await runPublishedEmbeddingJob(f.options(), { openSource: f.openSource }); + expect(result.phase).toBe('ready'); + expect(drain).toHaveBeenCalledTimes(1); + expect(drain.mock.invocationCallOrder[0]).toBeLessThan( + f.openSource.mock.invocationCallOrder[0], + ); + }); + + it('uses and releases the admitted local runtime, including initialization failure', async () => { + const f = await fixture(3); + const sidecar = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const initialize = vi.fn(async () => {}); + const embedBatch = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([1, 0, 0, 0])), + ); + const release = vi.fn(); + const reserve = vi + .spyOn(sidecar, 'reserveEmbeddingSidecar') + .mockResolvedValue({ initialize, embedBatch, release }); + const options = { + ...f.options(), + resources: { ...f.options().resources, modelBytes: 1024 * 1024 * 1024, modelHeapMb: 256 }, + }; + const dependencies = { identity: f.dependencies.identity, openSource: f.openSource }; + await expect( + runPublishedEmbeddingJob( + { ...options, resources: { ...options.resources, modelBytes: NaN } }, + dependencies, + ), + ).rejects.toThrow('resource reservation'); + expect(reserve).not.toHaveBeenCalled(); + expect((await loadMeta(f.root))?.embeddingJob).toBeUndefined(); + initialize.mockRejectedValueOnce(new Error('model init unavailable')); + await expect(runPublishedEmbeddingJob(options, dependencies)).rejects.toThrow( + 'model init unavailable', + ); + expect(embedBatch).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledTimes(1); + expect((await loadMeta(f.root))?.embeddingJob).toBeDefined(); + const result = await runPublishedEmbeddingJob(options, dependencies); + expect(result.phase).toBe('ready'); + expect(reserve).toHaveBeenCalledWith(expect.objectContaining({ threads: 1 }), 256); + expect(release).toHaveBeenCalledTimes(2); + expect(embedBatch.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + reserve.mockClear(); + expect((await runPublishedEmbeddingJob(options, dependencies)).alreadyComplete).toBe(true); + expect(reserve).not.toHaveBeenCalled(); + }); + + it('rejects a split vector pool below the native minimum before changing metadata', async () => { + const f = await fixture(3); + await f.run(); + await f.nextGeneration(); + const before = await loadMeta(f.root); + f.openSource.mockClear(); + await expect( + f.run({ resources: { ...f.options().resources, vectorPoolBytes: 8 * 1024 * 1024 } }), + ).rejects.toThrow('reuse reader do not fit'); + expect(await loadMeta(f.root)).toEqual(before); + expect(f.openSource).not.toHaveBeenCalled(); + }); + it('publishes complete vectors, then takes a no-model/no-writer fast path', async () => { const f = await fixture(); const result = await f.run(); diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 8a5121287..1d92955f6 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -59,6 +59,16 @@ describe('analysis resource admission', () => { ); }); + it('admits explicit embedding concurrency and includes sidecar heap within the model reservation', () => { + const budget = resolveAnalysisResourceBudget({ ...request, embeddingThreads: 8 }); + expect(budget.embedding.nativeThreads).toBe(8); + expect(budget.embedding.modelHeapMb).toBe(512); + expect(budget.embedding.modelHeapMb * MIB + 48 * MIB).toBeLessThan(budget.embedding.modelBytes); + expect(() => resolveAnalysisResourceBudget({ ...request, embeddingThreads: 25 })).toThrow( + 'CPU allowance', + ); + }); + it('admits the full working set of a supported 10 MiB node with a larger budget', () => { const budget = resolveAnalysisResourceBudget({ ...request, @@ -89,6 +99,12 @@ describe('analysis resource admission', () => { expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: 0 })).toThrow( 'graph pool bytes', ); + expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: MIB })).toThrow( + 'graph pool bytes', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, modelMemoryMb: 128 })).toThrow( + 'model reservation', + ); }); it.each([NaN, Infinity, -1, 0, 1.5, Number.MAX_SAFE_INTEGER + 1])( diff --git a/gitnexus/test/unit/embedding-sidecar-client.test.ts b/gitnexus/test/unit/embedding-sidecar-client.test.ts index 7e6943419..3a11cd75d 100644 --- a/gitnexus/test/unit/embedding-sidecar-client.test.ts +++ b/gitnexus/test/unit/embedding-sidecar-client.test.ts @@ -10,6 +10,7 @@ import type { SidecarRequest, SidecarResponse, } from '../../src/core/embeddings/embedding-sidecar-protocol.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; const embeddingsDir = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -144,6 +145,132 @@ describe('embedding sidecar client', () => { expect(forkMock).toHaveBeenCalledTimes(1); }); + it('reserves lazily, pins the admitted config, and reuses a compatible warm model', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const originalThreads = process.env.GITNEXUS_EMBEDDING_THREADS; + const originalNodeOptions = process.env.NODE_OPTIONS; + const first = await reserveEmbeddingSidecar(config, 512); + expect(forkMock).not.toHaveBeenCalled(); + await expect(reserveEmbeddingSidecar(config, 512)).rejects.toThrow('busy'); + try { + process.env.GITNEXUS_EMBEDDING_THREADS = '99'; + process.env.NODE_OPTIONS = '--max-old-space-size=4096'; + await first.initialize(); + const opts = forkMock.mock.calls[0][2]; + expect(opts.execArgv).toEqual( + expect.arrayContaining(['--max-old-space-size=512', '--max-semi-space-size=16']), + ); + expect(opts.env?.GITNEXUS_EMBEDDING_THREADS).toBeUndefined(); + expect(process.env.GITNEXUS_EMBEDDING_THREADS).toBe('99'); + expect(children[0].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: config, + }); + first.release(); + const second = await reserveEmbeddingSidecar(config, 512); + await second.initialize(); + await expect(second.embedBatch(['warm'])).resolves.toHaveLength(1); + second.release(); + expect(forkMock).toHaveBeenCalledTimes(1); + await expect(first.embedBatch(['released'])).rejects.toThrow('reservation was released'); + } finally { + first.release(); + if (originalThreads === undefined) delete process.env.GITNEXUS_EMBEDDING_THREADS; + else process.env.GITNEXUS_EMBEDDING_THREADS = originalThreads; + if (originalNodeOptions === undefined) delete process.env.NODE_OPTIONS; + else process.env.NODE_OPTIONS = originalNodeOptions; + } + }); + + it('waits for an idle old process to exit before assigning a different budget', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const first = await reserveEmbeddingSidecar(config, 512); + await first.initialize(); + first.release(); + children[0].kill.mockImplementation(() => true); + const next = reserveEmbeddingSidecar(config, 256); + await vi.waitFor(() => expect(children[0].kill).toHaveBeenCalled()); + expect(forkMock).toHaveBeenCalledTimes(1); + children[0].emit('close', null, 'SIGKILL'); + const second = await next; + await second.initialize(); + expect(forkMock).toHaveBeenCalledTimes(2); + expect(forkMock.mock.calls[1][2].execArgv).toContain('--max-old-space-size=256'); + second.release(); + }); + + it('keeps an aborted physical request reserved until the child actually responds', async () => { + const { reserveEmbeddingSidecar, drainEmbeddingSidecarForAnalysis } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const lease = await reserveEmbeddingSidecar(config, 512); + await lease.initialize(); + children[0].send = vi.fn(() => true); + const abort = new AbortController(); + const pendingBatch = lease.embedBatch(['still running'], { signal: abort.signal }); + await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled()); + const request = children[0].send.mock.calls[0][0]; + abort.abort(); + await expect(pendingBatch).rejects.toThrow(); + lease.release(); + await expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow('busy'); + await expect(drainEmbeddingSidecarForAnalysis()).rejects.toThrow('busy'); + expect(children[0].kill).not.toHaveBeenCalled(); + children[0].emit('message', { type: 'vectors', id: request.id, vectors: [[0.25, 0.5, 0.75]] }); + await drainEmbeddingSidecarForAnalysis(); + expect(children[0].kill).toHaveBeenCalledTimes(1); + }); + + it('does not certify a timed-out reap or allocate another job over the retained process', async () => { + vi.useFakeTimers(); + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const first = await reserveEmbeddingSidecar(config, 512); + await first.initialize(); + first.release(); + children[0].kill.mockImplementation(() => true); + const rejected = expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow( + 'native resource reservation', + ); + await vi.advanceTimersByTimeAsync(5_001); + await rejected; + await expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow( + 'native resource reservation', + ); + expect(forkMock).toHaveBeenCalledTimes(1); + children[0].emit('close', null, 'SIGKILL'); + const second = await reserveEmbeddingSidecar(config, 256); + second.release(); + }); + + it('rejects config changes while a job owns the runtime and preserves its allocation on respawn', async () => { + const { reserveEmbeddingSidecar, ensureEmbeddingSidecar, drainEmbeddingSidecarForAnalysis } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1, device: 'cpu' }); + const lease = await reserveEmbeddingSidecar(config, 512); + await expect(ensureEmbeddingSidecar({ forceDevice: 'cuda' })).rejects.toThrow('conflicts'); + expect(forkMock).not.toHaveBeenCalled(); + await lease.initialize(); + await expect(ensureEmbeddingSidecar({ embeddingConfig: { threads: 2 } })).rejects.toThrow( + 'conflicts', + ); + await expect(drainEmbeddingSidecarForAnalysis()).rejects.toThrow('busy'); + children[0].emit('close', 1, null); + await expect(lease.embedBatch(['retry'])).resolves.toHaveLength(1); + expect(forkMock.mock.calls[1][2].execArgv).toContain('--max-old-space-size=512'); + expect(children[1].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: config, + }); + lease.release(); + await drainEmbeddingSidecarForAnalysis(); + }); + it('does not fork on darwin/x64', async () => { const orig = { platform: process.platform, arch: process.arch }; Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); From 4890a0c09dad3d9ea761e5dcf4cae1034d4c4169 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 15:45:10 +0000 Subject: [PATCH 10/67] feat(status): separate graph and vector readiness Report embedding debt independently from graph content freshness. Read bounded durable progress without loading native runtimes, gate stale vector capability stamps, and allowlist additive phase counters on public SSE and ops payloads. --- gitnexus/src/cli/status.ts | 39 ++- gitnexus/src/core/analysis-progress.ts | 86 +++++++ gitnexus/src/core/embeddings/types.ts | 3 +- gitnexus/src/core/index-freshness.ts | 15 ++ gitnexus/src/core/run-analyze.ts | 7 +- gitnexus/src/mcp/resources.ts | 9 +- gitnexus/src/server/analyze-job.ts | 3 +- gitnexus/src/server/api.ts | 21 +- gitnexus/src/server/ops-snapshot.ts | 67 +++++- gitnexus/src/server/repo-projection.ts | 8 + .../src/storage/embedding-job-progress.ts | 48 ++++ gitnexus/src/storage/index-capabilities.ts | 52 ++++ .../test/unit/embedding-job-progress.test.ts | 224 ++++++++++++++++++ .../index-freshness-graph-collapse.test.ts | 33 +++ gitnexus/test/unit/ops-snapshot.test.ts | 30 +++ gitnexus/test/unit/repo-projection.test.ts | 17 ++ gitnexus/test/unit/resources.test.ts | 17 ++ gitnexus/test/unit/server-sse-payload.test.ts | 43 +++- .../test/unit/status-content-drift.test.ts | 38 +++ 19 files changed, 739 insertions(+), 21 deletions(-) create mode 100644 gitnexus/src/core/analysis-progress.ts create mode 100644 gitnexus/src/storage/embedding-job-progress.ts create mode 100644 gitnexus/src/storage/index-capabilities.ts create mode 100644 gitnexus/test/unit/embedding-job-progress.test.ts diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index 8316fc3ae..020b7dd96 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -39,7 +39,9 @@ import { analyzerRunnerIdentitiesEqual, resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js'; -import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { getFtsDisabledReason, FTS_DISABLED_MESSAGE } from '../core/search/fts-policy.js'; import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js'; import { @@ -218,6 +220,9 @@ export const statusCommand = async (options: StatusOptions = {}) => { runnerIdentity: meta.runnerIdentity ?? null, runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown', incompleteReasons, + graphIncompleteReasons: getGraphIncompleteReasons(meta), + embeddingProgress: await readEmbeddingJobProgress(storagePath, meta), + capabilities: await readIndexCapabilities(storagePath, meta), contentRetention: meta.contentRetention ?? 'full', }, current: sourceAvailable ? { commit: getCurrentCommit(entry.path) } : null, @@ -233,6 +238,14 @@ export const statusCommand = async (options: StatusOptions = {}) => { console.log(`Index storage: ${storagePath}`); console.log(`Indexed: ${new Date(meta.indexedAt).toLocaleString()}`); console.log(`Indexed commit: ${meta.lastCommit?.slice(0, 7)}`); + if (incompleteReasons.length) { + console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); + console.log( + `Graph incomplete reasons: ${JSON.stringify(payload.index.graphIncompleteReasons)}`, + ); + } + if (payload.index.embeddingProgress) + console.log(`Embedding progress: ${JSON.stringify(payload.index.embeddingProgress)}`); console.log( sourceAvailable ? 'Status: registered index (use status without --repo for working-tree freshness)' @@ -310,12 +323,15 @@ export const statusCommand = async (options: StatusOptions = {}) => { // commit comparison below then says whether it needs a re-analyze. Legacy/ // no-branch metas and detached HEAD also fall through to the flat index. let activeMeta = repo.meta; + let activeMetaDirectory = repo.storagePath; let workspaceLagsBranch = false; if (currentBranch && repo.meta.branch && currentBranch !== repo.meta.branch) { const { metaPath } = getStoragePaths(repo.repoPath, currentBranch, repo.storagePath); const branchMeta = await loadMeta(path.dirname(metaPath)); - if (branchMeta) activeMeta = branchMeta; - else workspaceLagsBranch = true; + if (branchMeta) { + activeMeta = branchMeta; + activeMetaDirectory = path.dirname(metaPath); + } else workspaceLagsBranch = true; } const currentRunnerIdentity = resolveAnalyzerRunnerIdentity(import.meta.url); @@ -324,10 +340,11 @@ export const statusCommand = async (options: StatusOptions = {}) => { currentRunnerIdentity, ); const incompleteReasons = getIndexIncompleteReasons(activeMeta); + const graphIncompleteReasons = getGraphIncompleteReasons(activeMeta); const metadataIsCurrent = currentCommit === activeMeta.lastCommit && runnerIdentityIsCurrent && - incompleteReasons.length === 0; + graphIncompleteReasons.length === 0; // A matching HEAD is not enough: `analyze` re-indexes changed content at the // same commit, so the files the index covers must still be compared against @@ -353,7 +370,10 @@ export const statusCommand = async (options: StatusOptions = {}) => { contentDrift.reason === 'no-file-hashes' && !isWorkingTreeDirty(repo.repoPath)); - const isUpToDate = metadataIsCurrent && contentIsCurrent; + const graphIsCurrent = metadataIsCurrent && contentIsCurrent; + const isUpToDate = graphIsCurrent && incompleteReasons.length === 0; + const embeddingProgress = await readEmbeddingJobProgress(activeMetaDirectory, activeMeta); + const capabilities = await readIndexCapabilities(activeMetaDirectory, activeMeta); // Shared sibling store (#3352): which graph this checkout reads, and any // pre-adoption index still sitting in /.gitnexus. const storeRoot = storeRootOfCheckoutSlot(repo.storagePath); @@ -386,10 +406,13 @@ export const statusCommand = async (options: StatusOptions = {}) => { index: { indexedAt: activeMeta.indexedAt, commit: activeMeta.lastCommit, - ...(activeMeta.capabilities ? { capabilities: activeMeta.capabilities } : {}), + ...(capabilities ? { capabilities } : {}), runnerIdentity: activeMeta.runnerIdentity ?? null, runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown', incompleteReasons, + graphIncompleteReasons, + graphStatus: graphIsCurrent ? 'up-to-date' : 'stale', + embeddingProgress, }, current: { commit: currentCommit, @@ -452,6 +475,10 @@ export const statusCommand = async (options: StatusOptions = {}) => { ); if (incompleteReasons.length > 0) { console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); + console.log(`Graph status: ${graphIsCurrent ? 'up-to-date' : 'stale'}`); + } + if (embeddingProgress) { + console.log(`Embedding progress: ${JSON.stringify(embeddingProgress)}`); } console.log(`${t('status.currentRunnerIdentity')}: ${JSON.stringify(currentRunnerIdentity)}`); if (contentDrift?.kind === 'current') { diff --git a/gitnexus/src/core/analysis-progress.ts b/gitnexus/src/core/analysis-progress.ts new file mode 100644 index 000000000..9a3659087 --- /dev/null +++ b/gitnexus/src/core/analysis-progress.ts @@ -0,0 +1,86 @@ +import type { EmbeddingJobState } from '../storage/embedding-job-journal.js'; + +export type GraphAnalysisPhase = 'pending' | 'building' | 'ready' | 'failed'; + +/** Counts describe durable acknowledgements. Active work is part of pending, + * never an additional category. Totals remain absent before manifest preflight. */ +export interface EmbeddingJobProgress { + phase: 'preparing' | 'embedding' | 'indexing' | 'ready' | 'failed' | 'unverified'; + totalUnits?: number; + completedUnits?: number; + reusedUnits?: number; + pendingUnits?: number; + failedUnits?: number; + activeUnit?: number | null; + totalNodes?: number; + completedNodes?: number; + reusedNodes?: number; + pendingNodes?: number; + failedNodes?: number; + chunks?: number; +} + +/** Additive fields shared by CLI callbacks, worker IPC, and HTTP progress. */ +export interface AnalysisPhaseProgress { + graphPhase?: GraphAnalysisPhase; + embeddingProgress?: EmbeddingJobProgress; + modelDownloadPercent?: number; +} + +export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJobProgress { + const counts = [ + state.totalUnits, + state.completedUnits, + state.reusedUnits, + state.failedUnits, + state.totalNodes, + state.completedNodes, + state.reusedNodes, + state.failedNodes, + state.chunks, + ]; + const pendingUnits = + state.totalUnits - state.completedUnits - state.reusedUnits - state.failedUnits; + const pendingNodes = + state.totalNodes - state.completedNodes - state.reusedNodes - state.failedNodes; + const activeUnit = state.active?.start?.unit ?? null; + if ( + counts.some((n) => !Number.isSafeInteger(n) || n < 0) || + pendingUnits < 0 || + pendingNodes < 0 || + state.active === undefined || + !['embedding', 'indexing', 'ready'].includes(state.phase) || + (state.active !== null && + (!Number.isSafeInteger(activeUnit) || + activeUnit === null || + activeUnit < 0 || + activeUnit >= state.totalUnits || + pendingUnits < 1 || + !Number.isSafeInteger(state.active.nodes) || + state.active.nodes < 1 || + state.active.nodes > pendingNodes)) || + (state.phase !== 'embedding' && + (pendingUnits || + pendingNodes || + state.failedUnits || + state.failedNodes || + state.active !== null)) + ) { + throw new Error('Embedding progress accounting is unverified'); + } + return { + phase: state.phase, + totalUnits: state.totalUnits, + completedUnits: state.completedUnits, + reusedUnits: state.reusedUnits, + pendingUnits, + failedUnits: state.failedUnits, + activeUnit, + totalNodes: state.totalNodes, + completedNodes: state.completedNodes, + reusedNodes: state.reusedNodes, + pendingNodes, + failedNodes: state.failedNodes, + chunks: state.chunks, + }; +} diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 65a4fd5f6..6ed1a8408 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -3,6 +3,7 @@ * * Type definitions for the embedding generation and semantic search system. */ +import type { AnalysisPhaseProgress } from '../analysis-progress.js'; export const LABEL_FUNCTION = 'Function' as const; export const LABEL_METHOD = 'Method' as const; @@ -206,7 +207,7 @@ export type EmbeddingPhase = /** * Progress information for the embedding pipeline */ -export interface EmbeddingProgress { +export interface EmbeddingProgress extends AnalysisPhaseProgress { phase: EmbeddingPhase; percent: number; modelDownloadPercent?: number; diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index 52e62e8d8..6ae7351a4 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -6,6 +6,7 @@ export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', 'embedding-count-unverified', + 'embedding-job-pending', 'graph-write-collapsed', 'scope-extraction-unverified', 'scope-extraction-failed', @@ -137,6 +138,7 @@ export function getIndexIncompleteReasons( RepoMeta, | 'incrementalInProgress' | 'embeddingCheckpoint' + | 'embeddingJob' | 'graphWriteCollapsed' | 'scopeExtractionFailures' | 'scopeExtractionReceipt' @@ -172,5 +174,18 @@ export function getIndexIncompleteReasons( : 'embedding-checkpoint-pending', ); } + if (meta?.embeddingJob) reasons.push('embedding-job-pending'); return reasons; } + +/** Embedding debt does not invalidate a published graph or require parsing. */ +export function getGraphIncompleteReasons( + meta: Parameters[0], +): IndexIncompleteReason[] { + return getIndexIncompleteReasons(meta).filter( + (reason) => + reason !== 'embedding-checkpoint-pending' && + reason !== 'embedding-count-unverified' && + reason !== 'embedding-job-pending', + ); +} diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 9397374fc..ebd4dc37f 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -457,7 +457,12 @@ function collectSpringBeanDeclarationDriftFiles( // --------------------------------------------------------------------------- export interface AnalyzeCallbacks { - onProgress: (phase: string, percent: number, message: string) => void; + onProgress: ( + phase: string, + percent: number, + message: string, + details?: import('./analysis-progress.js').AnalysisPhaseProgress, + ) => void; onLog?: (message: string) => void; } diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index 6a3de9431..49dc3746f 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -9,7 +9,9 @@ import type { LocalBackend } from './local/local-backend.js'; import { checkStaleness } from './staleness.js'; import { loadMeta } from '../storage/repo-manager.js'; import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from '../core/analyzer-identity.js'; -import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { checkoutIsDirectory, contentRetentionFromMeta, @@ -381,6 +383,11 @@ async function getContextResource(backend: LocalBackend, repoName?: string): Pro lines.push(` source_available: ${JSON.stringify(sourceAvailable)}`); lines.push(` runner_identity: ${JSON.stringify(freshMeta?.runnerIdentity ?? null)}`); lines.push(` incomplete_reasons: ${JSON.stringify(incompleteReasons)}`); + lines.push(` graph_incomplete_reasons: ${JSON.stringify(getGraphIncompleteReasons(freshMeta))}`); + const embeddingProgress = await readEmbeddingJobProgress(repo.storagePath, freshMeta); + if (embeddingProgress) lines.push(` embedding_progress: ${JSON.stringify(embeddingProgress)}`); + const capabilities = await readIndexCapabilities(repo.storagePath, freshMeta); + if (capabilities) lines.push(` capabilities: ${JSON.stringify(capabilities)}`); lines.push(` spring_actuator: ${JSON.stringify(freshMeta?.springActuator ?? null)}`); // Surfaced beside the Actuator flag, and it matters more than that one does: // Actuator only annotates nodes the source pass already found, whereas diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index 2ab767e7d..5a6319ac0 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -12,8 +12,9 @@ import { randomUUID } from 'crypto'; import { EventEmitter } from 'events'; import type { ChildProcess } from 'child_process'; +import type { AnalysisPhaseProgress } from '../core/analysis-progress.js'; -export interface AnalyzeJobProgress { +export interface AnalyzeJobProgress extends AnalysisPhaseProgress { phase: string; percent: number; message: string; diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 5a819edcf..1eea07846 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -17,6 +17,8 @@ import { } from '../storage/index-lock.js'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; import { resolveGraphPath } from '../storage/shared-store.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { reclaimAfterSlotRemoval, removeCheckoutStorage, @@ -1270,15 +1272,24 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // Timed out waiting for an active analysis job if (respondIfAnalysisPending(entry, res)) return; const meta = await loadMeta(entry.storagePath); - const [staleness, availability] = await Promise.all([ + const [staleness, availability, embeddingProgress, capabilities] = await Promise.all([ checkStalenessAsync(entry.path, resolveLastCommit(entry, meta)), getSourceAvailability(entry, meta), + readEmbeddingJobProgress(entry.storagePath, meta), + readIndexCapabilities(entry.storagePath, meta), ]); res.json( - projectRepoDetail(entry, meta, staleness, { - contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta), - sourceAvailable: availability.available, - }), + projectRepoDetail( + entry, + meta, + staleness, + { + contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta), + sourceAvailable: availability.available, + }, + embeddingProgress, + capabilities, + ), ); } catch (err: any) { if (sendStorageRequirementHttp(err, res)) return; diff --git a/gitnexus/src/server/ops-snapshot.ts b/gitnexus/src/server/ops-snapshot.ts index 18ed36f2d..f87512208 100644 --- a/gitnexus/src/server/ops-snapshot.ts +++ b/gitnexus/src/server/ops-snapshot.ts @@ -13,6 +13,46 @@ import { } from './analyze-job.js'; import { publicRepoId } from './public-repo-id.js'; import { escapeRegExp } from './validation.js'; +import type { EmbeddingJobProgress } from '../core/analysis-progress.js'; + +const publicEmbeddingProgress = ( + progress: EmbeddingJobProgress | undefined, +): EmbeddingJobProgress | undefined => { + if ( + !progress || + !['preparing', 'embedding', 'indexing', 'ready', 'failed', 'unverified'].includes( + progress.phase, + ) + ) + return undefined; + const safe: EmbeddingJobProgress = { phase: progress.phase }; + // Explicit numeric allowlist: never spread the worker/job object onto a + // public feed. It may also contain source paths, identities, or credentials. + for (const key of [ + 'totalUnits', + 'completedUnits', + 'reusedUnits', + 'pendingUnits', + 'failedUnits', + 'totalNodes', + 'completedNodes', + 'reusedNodes', + 'pendingNodes', + 'failedNodes', + 'chunks', + ] as const) { + const value = progress[key]; + if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) safe[key] = value; + } + if ( + progress.activeUnit === null || + (typeof progress.activeUnit === 'number' && + Number.isSafeInteger(progress.activeUnit) && + progress.activeUnit >= 0) + ) + safe.activeUnit = progress.activeUnit; + return safe; +}; export interface OpsJobView { id: string; @@ -227,11 +267,28 @@ export const redactPublicText = (text: string, known?: Array export const publicOpsProgress = ( progress: AnalyzeJobProgress, known?: Array, -): AnalyzeJobProgress => ({ - phase: progress.phase, - percent: progress.percent, - message: redactPublicText(progress.message, known), -}); +): AnalyzeJobProgress => { + const result: AnalyzeJobProgress = { + phase: progress.phase, + percent: progress.percent, + message: redactPublicText(progress.message, known), + }; + if ( + progress.graphPhase && + ['pending', 'building', 'ready', 'failed'].includes(progress.graphPhase) + ) + result.graphPhase = progress.graphPhase; + const embeddingProgress = publicEmbeddingProgress(progress.embeddingProgress); + if (embeddingProgress) result.embeddingProgress = embeddingProgress; + if ( + typeof progress.modelDownloadPercent === 'number' && + Number.isFinite(progress.modelDownloadPercent) && + progress.modelDownloadPercent >= 0 && + progress.modelDownloadPercent <= 100 + ) + result.modelDownloadPercent = progress.modelDownloadPercent; + return result; +}; export const serializeOpsJob = ( job: AnalyzeJob, diff --git a/gitnexus/src/server/repo-projection.ts b/gitnexus/src/server/repo-projection.ts index bfb0171f4..7605a3d86 100644 --- a/gitnexus/src/server/repo-projection.ts +++ b/gitnexus/src/server/repo-projection.ts @@ -17,6 +17,8 @@ import { import type { ContentRetention, RepoMeta } from '../storage/repo-meta.js'; import type { RegistryEntry } from '../storage/repo-manager.js'; import { publicRepoId } from './public-repo-id.js'; +import { getGraphIncompleteReasons, getIndexIncompleteReasons } from '../core/index-freshness.js'; +import type { EmbeddingJobProgress } from '../core/analysis-progress.js'; /** Retention + checkout facts computed by the route (see getSourceAvailability). */ export interface RepoProjectionSource { @@ -85,6 +87,8 @@ export const projectRepoDetail = ( meta: RepoMeta | null | undefined, staleness: StalenessInfo, source: RepoProjectionSource, + embeddingProgress?: EmbeddingJobProgress, + capabilities?: RepoMeta['capabilities'], ) => ({ name: entry.name, repoPath: entry.path, @@ -95,6 +99,10 @@ export const projectRepoDetail = ( branch: meta?.branch ?? entry.branch, contentRetention: source.contentRetention, sourceAvailable: source.sourceAvailable, + incompleteReasons: getIndexIncompleteReasons(meta), + graphIncompleteReasons: getGraphIncompleteReasons(meta), + embeddingProgress, + capabilities, ...stalenessField(staleness), }); diff --git a/gitnexus/src/storage/embedding-job-progress.ts b/gitnexus/src/storage/embedding-job-progress.ts new file mode 100644 index 000000000..6b9d60153 --- /dev/null +++ b/gitnexus/src/storage/embedding-job-progress.ts @@ -0,0 +1,48 @@ +/** Observation only: bounded journal read, no source scan, native DB, or model. */ +import path from 'node:path'; +import { + embeddingProgressFromState, + type EmbeddingJobProgress, +} from '../core/analysis-progress.js'; +import type { RepoMeta } from './repo-meta.js'; +import type { EmbeddingJobState } from './embedding-job-journal.js'; +import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js'; +import { embeddingGenerationDirectory } from './vector-generation.js'; + +export async function readEmbeddingJobProgress( + metadataDirectory: string, + metadata: RepoMeta | null | undefined, +): Promise { + const reference = metadata?.embeddingJob; + if (!reference) return undefined; + try { + if ( + reference.version !== 1 || + metadata?.graphGeneration?.version !== 1 || + !/^[a-f0-9]{32}$/.test(reference.graphGeneration) || + reference.graphGeneration !== metadata.graphGeneration.id + ) + throw new Error('Job graph mismatch'); + const directory = embeddingGenerationDirectory(metadataDirectory, reference.identity); + // Validate every job-owned directory before following the path. In + // particular, a symlinked embeddings parent must not redirect this read. + await embeddingJobDirectory(metadataDirectory); + await embeddingJobDirectory(path.dirname(directory)); + const state = (await readEmbeddingJobRecord(directory, 'job.json')) as EmbeddingJobState; + if ( + !state || + state.version !== 1 || + state.identity !== reference.identity || + !/^[a-f0-9]{64}$/.test(state.manifestDigest) + ) + throw new Error('Job identity mismatch'); + const progress = embeddingProgressFromState(state); + // A journal alone cannot publish a vector capability. If the metadata + // still points at pending work, even a ready journal awaits publication. + if (progress.phase === 'ready') progress.phase = 'indexing'; + return progress; + } catch { + // Missing/corrupt/unreadable work is not a zero-work or completed job. + return { phase: 'unverified' }; + } +} diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts new file mode 100644 index 000000000..a959d30f7 --- /dev/null +++ b/gitnexus/src/storage/index-capabilities.ts @@ -0,0 +1,52 @@ +/** Published capability receipts, gated by current routing and artifact ownership. + * Runtime query probes may further degrade these facts; status never loads a DB. */ +import path from 'node:path'; +import type { RepoMeta } from './repo-meta.js'; +import { assertOwnedVectorGeneration, selectVectorGeneration } from './vector-generation.js'; +import { getExactScanLimit } from '../core/platform/capabilities.js'; + +export async function readIndexCapabilities( + directory: string, + metadata: RepoMeta | null | undefined, +): Promise { + if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob) + return metadata?.capabilities; + const exactScanLimit = getExactScanLimit(); + let vectorSearch: NonNullable['vectorSearch']; + try { + const selection = selectVectorGeneration( + directory, + metadata, + path.resolve(directory, metadata?.graphPath ?? 'lbug'), + exactScanLimit, + ); + if (selection.kind === 'ready') { + await assertOwnedVectorGeneration(directory, selection.generation.identity); + vectorSearch = { + provider: selection.generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: selection.generation.indexKind === 'hnsw' ? 'vector-index' : 'exact-scan', + exactScanLimit, + }; + } else { + vectorSearch = { + provider: 'ladybugdb-vector', + status: 'unavailable', + exactScanLimit, + reason: + selection.kind === 'unavailable' ? selection.reason : 'vector-generation-unverified', + }; + } + } catch { + vectorSearch = { + provider: 'ladybugdb-vector', + status: 'unavailable', + exactScanLimit, + reason: 'vector-artifact-unavailable', + }; + } + return { + graph: metadata?.capabilities?.graph ?? { provider: 'ladybugdb', status: 'unavailable' }, + fts: metadata?.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' }, + vectorSearch, + }; +} diff --git a/gitnexus/test/unit/embedding-job-progress.test.ts b/gitnexus/test/unit/embedding-job-progress.test.ts new file mode 100644 index 000000000..a3226d3a7 --- /dev/null +++ b/gitnexus/test/unit/embedding-job-progress.test.ts @@ -0,0 +1,224 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { readEmbeddingJobProgress } from '../../src/storage/embedding-job-progress.js'; +import { writeEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { embeddingProgressFromState } from '../../src/core/analysis-progress.js'; +import { readIndexCapabilities } from '../../src/storage/index-capabilities.js'; + +const identity = 'a'.repeat(64); +const generation = 'b'.repeat(32); +const manifest = { + version: 1 as const, + sourceIdentity: 'c'.repeat(64), + digest: 'd'.repeat(64), + bytes: 30, + totalNodes: 6, + totalUnits: 3, + maximumUnitBytes: 100, +}; +const meta: RepoMeta = { + repoPath: '/repo', + lastCommit: 'head', + indexedAt: '', + graphGeneration: { version: 1, id: generation, coveredContent: 'e'.repeat(64) }, + embeddingJob: { version: 1, identity, graphGeneration: generation }, +}; +const unit = (n: number, origin: 'manifest' | 'failed' = 'manifest') => ({ + start: { unit: n, offset: n * 10 }, + next: { unit: n + 1, offset: (n + 1) * 10 }, + nodes: 2, + origin, +}); +const directories: string[] = []; +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-job-progress-')); + directories.push(root); + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + return { + root, + directory, + journal: await EmbeddingJobJournal.create(directory, identity, manifest), + }; +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('bounded durable embedding progress', () => { + it('gates a persisted vector-index stamp before filesystem work when the new job is pending', async () => { + const f = await fixture(); + const previous: NonNullable = { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'available' }, + vectorSearch: { provider: 'ladybugdb-vector', status: 'vector-index', exactScanLimit: 10000 }, + }; + const stat = vi.spyOn(fs, 'lstat'); + expect(await readIndexCapabilities(f.root, { ...meta, capabilities: previous })).toMatchObject({ + graph: previous.graph, + fts: previous.fts, + vectorSearch: { status: 'unavailable', reason: 'embedding-pending' }, + }); + expect(stat).not.toHaveBeenCalled(); + expect( + await readIndexCapabilities('/missing', { + repoPath: '/repo', + indexedAt: '', + lastCommit: '', + capabilities: previous, + }), + ).toBe(previous); + }); + + it('reports missing published vector artifacts without opening a native database', async () => { + const f = await fixture(); + const published = { + ...meta, + embeddingJob: undefined, + vectorGeneration: { + version: 1 as const, + identity, + graphGeneration: generation, + vectorIdentity: 'c'.repeat(64), + inputIdentity: 'd'.repeat(64), + dimensions: 3, + nodes: 6, + chunks: 7, + indexKind: 'hnsw' as const, + }, + }; + expect(await readIndexCapabilities(f.root, published)).toMatchObject({ + vectorSearch: { status: 'unavailable', reason: 'vector-artifact-unavailable' }, + }); + expect( + await readIndexCapabilities(f.root, { + ...published, + vectorGeneration: { ...published.vectorGeneration, graphGeneration: 'f'.repeat(32) }, + }), + ).toMatchObject({ + vectorSearch: { status: 'unavailable', reason: 'vector-generation-mismatch' }, + }); + }); + it('does no filesystem work for legacy or graph-only metadata', async () => { + const stat = vi.spyOn(fs, 'lstat'); + expect(await readEmbeddingJobProgress('/missing', null)).toBeUndefined(); + expect( + await readEmbeddingJobProgress('/missing', { ...meta, embeddingJob: undefined }), + ).toBeUndefined(); + expect(stat).not.toHaveBeenCalled(); + }); + + it('counts durable completions, reuse, failures and active retries without double counting', async () => { + const f = await fixture(); + await f.journal.begin(unit(0)); + await f.journal.acknowledge({ reusedNodes: 1, chunks: 3 }); + await f.journal.begin(unit(1)); + await f.journal.fail('provider'); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 1, + failedUnits: 1, + activeUnit: null, + totalNodes: 6, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 2, + failedNodes: 2, + chunks: 3, + }); + await f.journal.begin(unit(1, 'failed')); + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + pendingUnits: 2, + failedUnits: 0, + activeUnit: 1, + pendingNodes: 4, + failedNodes: 0, + }); + await f.journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + await f.journal.begin(unit(2)); + await f.journal.acknowledge({ reusedNodes: 0, chunks: 2 }); + await f.journal.indexing(); + await f.journal.ready(); + // Progress never certifies publication merely because the journal says ready. + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + phase: 'indexing', + completedUnits: 2, + reusedUnits: 1, + pendingUnits: 0, + failedUnits: 0, + completedNodes: 3, + reusedNodes: 3, + pendingNodes: 0, + failedNodes: 0, + }); + }); + + it('does not scan a manifest or source database to observe progress', async () => { + const f = await fixture(); + // No manifest.jsonl or vectors file exists. Observing a journal still works. + const open = vi.spyOn(fs, 'open'); + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + totalNodes: 6, + pendingNodes: 6, + }); + expect(open.mock.calls.map((call) => path.basename(String(call[0])))).toEqual(['job.json']); + }); + + it('reports missing, corrupt and oversized receipts as unverified, never zero work', async () => { + const f = await fixture(); + const file = path.join(f.directory, 'job.json'); + await fs.unlink(file); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await fs.writeFile(file, '{broken'); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await fs.writeFile(file, 'x'.repeat(33 * 1024)); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + }); + + it('refuses foreign graph identities and symlinked job ancestors', async () => { + const f = await fixture(); + expect( + await readEmbeddingJobProgress(f.root, { + ...meta, + embeddingJob: { version: 1, identity, graphGeneration: 'f'.repeat(32) }, + }), + ).toEqual({ phase: 'unverified' }); + const moved = path.join(f.root, 'preserved'); + await fs.rename(path.join(f.root, 'embeddings'), moved); + await fs.symlink( + moved, + path.join(f.root, 'embeddings'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + }); + + it('rejects impossible accounting and drops unknown journal fields', async () => { + const f = await fixture(); + await writeEmbeddingJobRecord(f.directory, 'job.json', { + ...f.journal.snapshot(), + completedUnits: 4, + }); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await writeEmbeddingJobRecord(f.directory, 'job.json', { + ...f.journal.snapshot(), + sourcePath: '/secret', + token: 'private', + }); + expect(JSON.stringify(await readEmbeddingJobProgress(f.root, meta))).not.toMatch( + /secret|private|identity|manifest/, + ); + expect(() => embeddingProgressFromState({ ...f.journal.snapshot(), phase: 'ready' })).toThrow( + 'unverified', + ); + }); +}); diff --git a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts index d2c32a280..99d53b499 100644 --- a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts +++ b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts @@ -17,11 +17,44 @@ import { describe, it, expect } from 'vitest'; import { detectGraphWriteCollapse, getIndexIncompleteReasons, + getGraphIncompleteReasons, GRAPH_WRITE_COLLAPSE_MIN_EDGES, GRAPH_WRITE_COLLAPSE_RATIO, INDEX_INCOMPLETE_REASONS, } from '../../src/core/index-freshness.js'; +it('keeps graph failures visible while separating legacy and durable embedding debt', () => { + const metadata = { + scopeExtractionReceipt: 1 as const, + embeddingJob: { + version: 1 as const, + identity: 'a'.repeat(64), + graphGeneration: 'b'.repeat(32), + }, + graphWriteCollapsed: { expected: 100, persisted: 0 }, + }; + expect(getIndexIncompleteReasons(metadata)).toEqual([ + 'graph-write-collapsed', + 'embedding-job-pending', + ]); + expect(getGraphIncompleteReasons(metadata)).toEqual(['graph-write-collapsed']); + expect( + getGraphIncompleteReasons({ + scopeExtractionReceipt: 1, + embeddingCheckpoint: { + at: '', + model: 'fixture', + dimensions: 3, + provider: 'local', + nodesProcessed: 0, + totalNodes: 1, + chunksProcessed: 0, + }, + }), + ).toEqual([]); + expect(getGraphIncompleteReasons(null)).toEqual(['scope-extraction-unverified']); +}); + describe('detectGraphWriteCollapse (B2 detection)', () => { it('flags the reported field failure (23009 built, 2170 persisted)', () => { expect(detectGraphWriteCollapse(23009, 2170)).toEqual({ diff --git a/gitnexus/test/unit/ops-snapshot.test.ts b/gitnexus/test/unit/ops-snapshot.test.ts index 298b823e1..6c55bad90 100644 --- a/gitnexus/test/unit/ops-snapshot.test.ts +++ b/gitnexus/test/unit/ops-snapshot.test.ts @@ -5,8 +5,38 @@ import { isGitNexusVercelOrigin, serializeOpsJob, summarizeOpsLane, + publicOpsProgress, } from '../../src/server/ops-snapshot.js'; import { publicRepoId } from '../../src/server/public-repo-id.js'; +import type { AnalyzeJobProgress } from '../../src/server/analyze-job.js'; + +it('copies only enumerated phases and finite numeric progress fields', () => { + const input = { + phase: 'embedding', + percent: 90, + message: 'Embedding', + graphPhase: '/private/path', + modelDownloadPercent: Infinity, + embeddingProgress: { + phase: 'embedding', + totalUnits: 'secret', + completedUnits: -1, + reusedUnits: NaN, + failedUnits: 1.5, + pendingUnits: 2, + activeUnit: '/secret', + chunks: 7, + identity: 'hidden', + endpoint: 'https://secret', + }, + } as unknown as AnalyzeJobProgress; + expect(publicOpsProgress(input)).toEqual({ + phase: 'embedding', + percent: 90, + message: 'Embedding', + embeddingProgress: { phase: 'embedding', pendingUnits: 2, chunks: 7 }, + }); +}); describe('serializeOpsJob / summarizeOpsLane', () => { let manager: JobManager; diff --git a/gitnexus/test/unit/repo-projection.test.ts b/gitnexus/test/unit/repo-projection.test.ts index e58f27d8d..6d9a2a44c 100644 --- a/gitnexus/test/unit/repo-projection.test.ts +++ b/gitnexus/test/unit/repo-projection.test.ts @@ -171,6 +171,23 @@ describe('projectRepoListEntry — GET /api/repos', () => { }); describe('projectRepoDetail — GET /api/repo', () => { + it('reports graph completeness separately from pending vector work', () => { + const metadata = meta({ + scopeExtractionReceipt: 1, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + const out = projectRepoDetail(entry(), metadata, FRESH, FULL_SOURCE, { + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 1, + failedUnits: 1, + }); + expect(out.graphIncompleteReasons).toEqual([]); + expect(out.incompleteReasons).toEqual(['embedding-job-pending']); + expect(out.embeddingProgress).toMatchObject({ failedUnits: 1, pendingUnits: 1 }); + }); it('returns lastCommit and branch, which the route used to drop', () => { const out = projectRepoDetail(entry({ branch: 'master' }), null, FRESH, FULL_SOURCE); expect(out.lastCommit).toBe(entry().lastCommit); diff --git a/gitnexus/test/unit/resources.test.ts b/gitnexus/test/unit/resources.test.ts index 4d50a610d..c25619d0e 100644 --- a/gitnexus/test/unit/resources.test.ts +++ b/gitnexus/test/unit/resources.test.ts @@ -566,6 +566,23 @@ describe('context resource freshness after out-of-process analyze (#2438)', () = ); }); + it('separates a complete graph from an unverified pending embedding journal', async () => { + loadMetaMock.mockResolvedValue({ + repoPath: '/tmp/test-repo', + lastCommit: 'current-head', + indexedAt: '', + scopeExtractionReceipt: 1, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + const result = await readResource( + 'gitnexus://repo/test-project/context', + createMockBackend({ context: CONTEXT }), + ); + expect(result).toContain('incomplete_reasons: ["embedding-job-pending"]'); + expect(result).toContain('graph_incomplete_reasons: []'); + expect(result).toContain('embedding_progress: {"phase":"unverified"}'); + }); + it('exposes Spring Actuator index metadata to agents', async () => { loadMetaMock.mockResolvedValue({ repoPath: '/tmp/test-repo', diff --git a/gitnexus/test/unit/server-sse-payload.test.ts b/gitnexus/test/unit/server-sse-payload.test.ts index 0b11766c2..94384026e 100644 --- a/gitnexus/test/unit/server-sse-payload.test.ts +++ b/gitnexus/test/unit/server-sse-payload.test.ts @@ -16,7 +16,7 @@ * MCP wiring, which is what made this file cost ~25s to import (#2790 review). */ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { JobManager } from '../../src/server/analyze-job.js'; +import type { JobManager, AnalyzeJobProgress } from '../../src/server/analyze-job.js'; import { publicRepoId } from '../../src/server/public-repo-id.js'; import { startSSEHarness, terminalFrame, type SSEHarness } from '../helpers/sse-harness.js'; @@ -37,6 +37,47 @@ describe('mountSSEProgress terminal payload', () => { afterEach(() => harness.close()); + it('keeps graph-ready nonterminal, forwards safe counts, and reports the eventual vector failure once', async () => { + const job = manager.createJob({ repoPath: REPO_PATH }); + manager.updateJob(job.id, { status: 'analyzing' }); + const response = await fetch(`${baseUrl}/api/analyze/${job.id}/progress`); + manager.updateJob(job.id, { + progress: { + phase: 'graph-ready', + percent: 90, + message: 'Graph available', + graphPhase: 'ready', + embeddingProgress: { + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 2, + failedUnits: 0, + activeUnit: 1, + sourcePath: '/private/source', + identity: 'private-key', + }, + modelDownloadPercent: 100, + secret: 'hidden-token', + } as AnalyzeJobProgress, + }); + expect(manager.getJob(job.id)?.status).toBe('analyzing'); + manager.updateJob(job.id, { status: 'failed', error: 'Embedding endpoint unavailable' }); + const body = await response.text(); + expect(body).toContain('"graphPhase":"ready"'); + expect(body).toContain('"activeUnit":1'); + expect(body).toContain('"pendingUnits":2'); + expect(body).toContain('"modelDownloadPercent":100'); + expect(body).not.toMatch(/private|hidden-token|sourcePath|identity/); + expect(body.match(/event: failed/g)).toHaveLength(1); + expect(body).not.toContain('event: complete'); + expect(terminalFrame(body, 'failed')).toEqual({ + repoName: 'reels', + error: 'Embedding endpoint unavailable', + }); + }); + it('already-terminal replay includes repoName and omits repoPath', async () => { const job = manager.createJob({ repoPath: REPO_PATH }); manager.updateJob(job.id, { status: 'complete', repoName: REPO_NAME }); diff --git a/gitnexus/test/unit/status-content-drift.test.ts b/gitnexus/test/unit/status-content-drift.test.ts index f6b131f20..65131dd14 100644 --- a/gitnexus/test/unit/status-content-drift.test.ts +++ b/gitnexus/test/unit/status-content-drift.test.ts @@ -115,6 +115,44 @@ afterEach(() => { }); describe('status freshness from per-file drift (#3077)', () => { + it('checks graph content despite pending embeddings and preserves the aggregate incomplete verdict', async () => { + vi.mocked(loadMeta).mockResolvedValue({ + ...repoWithCoverage.meta, + graphGeneration: { version: 1, id: 'b'.repeat(32), coveredContent: 'c'.repeat(64) }, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + vi.mocked(detectIndexContentDrift).mockResolvedValue({ kind: 'current', coveredFileCount: 1 }); + await statusCommand({ json: true }); + expect(detectIndexContentDrift).toHaveBeenCalled(); + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + index: { + graphStatus: 'up-to-date', + graphIncompleteReasons: [], + incompleteReasons: ['embedding-job-pending'], + embeddingProgress: { phase: 'unverified' }, + }, + }); + }); + + it('does not certify a drifted graph while embeddings are pending', async () => { + vi.mocked(loadMeta).mockResolvedValue({ + ...repoWithCoverage.meta, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + vi.mocked(detectIndexContentDrift).mockResolvedValue({ + kind: 'drifted', + changed: ['a.js'], + added: [], + deleted: [], + }); + await statusCommand({ json: true }); + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + index: { graphStatus: 'stale' }, + contentDrift: { status: 'drifted' }, + }); + }); it('is up-to-date when every covered file matches, despite a dirty working tree', async () => { // The reported case: one modified file outside the index's coverage. The // old repo-wide check called this stale and `analyze` could not clear it. From 351ca6dd1556651f2fbfc59455d220357417e92c Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 16:06:36 +0000 Subject: [PATCH 11/67] fix(embeddings): recover interrupted job setup Recover an owned empty native catalog after interrupted bootstrap, and rebuild damaged failure accounting without repeating completed provider work. Preserve foreign artifacts and propagate filesystem errors. --- gitnexus/src/core/embeddings/embedding-job.ts | 4 +- gitnexus/src/core/lbug/vector-store.ts | 49 ++++--- gitnexus/src/storage/embedding-job-journal.ts | 64 +++++++-- .../helpers/interrupted-vector-bootstrap.mjs | 13 ++ .../embedding-job-controller.test.ts | 27 ++++ .../embedding-vector-store.test.ts | 131 ++++++++++++++++++ .../test/unit/embedding-job-journal.test.ts | 43 ++++++ 7 files changed, 302 insertions(+), 29 deletions(-) create mode 100644 gitnexus/test/helpers/interrupted-vector-bootstrap.mjs diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index aec5655a8..931aaada3 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -255,13 +255,15 @@ export async function runPublishedEmbeddingJob( ? await loadEmbeddingWorkManifest(directory, identities.sourceIdentity) : await createEmbeddingWorkManifest(directory, identities.sourceIdentity, await getSource()); const hasStore = await existsOwnedFile(path.join(directory, 'vectors')); - if (!hasStore && (await existsOwnedFile(path.join(directory, 'job.json')))) + const hasJournal = await existsOwnedFile(path.join(directory, 'job.json')); + if (!hasStore && hasJournal) throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); store = await VectorStore.open({ directory, identity: identities.identity, dimensions: vector.dimensions, mode: hasStore ? 'resume' : 'create', + recoverEmptyBootstrap: !hasJournal, ...storeResources, }); const journal = await EmbeddingJobJournal.recover(directory, identities.identity, manifest); diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index bdfa735ec..b82d67efa 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -40,6 +40,9 @@ export interface VectorStoreOptions { bufferPoolBytes: number; threads: number; mode: 'create' | 'resume' | 'read'; + /** Only before a scheduling journal exists: an interrupted initial DDL + * transaction may leave an owned database with a verified empty catalog. */ + recoverEmptyBootstrap?: boolean; /** Maximum JS vector batch retained by the caller and passed into native. */ batchBytes: number; } @@ -83,7 +86,7 @@ export class VectorStore { throw new Error('Invalid vector-store resource reservation or identity'); const directory = await embeddingJobDirectory(options.directory); const file = path.join(directory, 'vectors'); - for (const suffix of ['', '.wal', '.lock']) { + for (const suffix of ['', '.wal', '.shadow', '.lock']) { try { const stat = await fs.lstat(file + suffix); if (!stat.isFile() || stat.isSymbolicLink()) @@ -105,24 +108,32 @@ export class VectorStore { }); const store = new VectorStore(handle, Object.freeze({ ...options, directory })); try { - if (options.mode === 'create') { - await store.query( - `CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${options.dimensions}], contentHash STRING, PRIMARY KEY(id))`, - ); - await store.query( - 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', - ); - await store.query( - 'CREATE NODE TABLE EmbeddingArtifact (id STRING, identity STRING, schema STRING, dimensions INT64, complete BOOL, indexKind STRING, PRIMARY KEY(id))', - ); - await store.query( - "CREATE (:EmbeddingArtifact {id:'artifact', identity:$identity, schema:$schema, dimensions:$dimensions, complete:false, indexKind:'pending'})", - { - identity: options.identity, - schema: VECTOR_STORE_SCHEMA, - dimensions: options.dimensions, - }, - ); + const emptyBootstrap = + options.mode === 'resume' && + options.recoverEmptyBootstrap === true && + (await store.query('CALL show_tables() RETURN *')).length === 0; + if (options.mode === 'create' || emptyBootstrap) { + // DDL is transactional in LadybugDB. A crash must expose either the + // complete schema/identity or an empty catalog, never half a bootstrap. + await store.transaction(async () => { + await store.query( + `CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${options.dimensions}], contentHash STRING, PRIMARY KEY(id))`, + ); + await store.query( + 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', + ); + await store.query( + 'CREATE NODE TABLE EmbeddingArtifact (id STRING, identity STRING, schema STRING, dimensions INT64, complete BOOL, indexKind STRING, PRIMARY KEY(id))', + ); + await store.query( + "CREATE (:EmbeddingArtifact {id:'artifact', identity:$identity, schema:$schema, dimensions:$dimensions, complete:false, indexKind:'pending'})", + { + identity: options.identity, + schema: VECTOR_STORE_SCHEMA, + dimensions: options.dimensions, + }, + ); + }); await store.query('CHECKPOINT'); } else { const rows = await store.query( diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts index 2f5c43da7..0025606b7 100644 --- a/gitnexus/src/storage/embedding-job-journal.ts +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -56,6 +56,16 @@ const integer = (value: unknown): value is number => const filename = (unit: number): string => `failed-${String(unit).padStart(16, '0')}.json`; const clone = (value: T): T => structuredClone(value); +/** Operational failures (permissions, I/O) must not be treated as lost state. */ +function damagedRecord(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + if (code) return code === 'ENOENT' || code === 'ERR_ENCODING_INVALID_ENCODED_DATA'; + return ( + error instanceof SyntaxError || + /checksum|Malformed|Invalid|UTF-8|receipt exceeds/i.test(String(error)) + ); +} + function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): void { if ( !intent || @@ -215,14 +225,7 @@ export class EmbeddingJobJournal { try { value = await readEmbeddingJobRecord(canonical, 'job.json'); } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code && code !== 'ENOENT' && code !== 'ERR_ENCODING_INVALID_ENCODED_DATA') throw error; - if ( - !code && - !(error instanceof SyntaxError) && - !/checksum|Malformed|Invalid|UTF-8|receipt exceeds/i.test(String(error)) - ) - throw error; + if (!damagedRecord(error)) throw error; } if (value && typeof value === 'object') { const state = value as Partial; @@ -232,12 +235,23 @@ export class EmbeddingJobJournal { (state.manifestDigest && state.manifestDigest !== manifest.digest) ) throw new Error('Embedding job identity mismatch; recovery refused'); + let journal: EmbeddingJobJournal | undefined; try { validateState(value, identity, manifest); - return new EmbeddingJobJournal(canonical, value, clone(manifest)); + journal = new EmbeddingJobJournal(canonical, value, clone(manifest)); } catch { /* Matching identity but damaged accounting requires reconstruction. */ } + if (journal) { + try { + await journal.verifyFailedRecords(); + return journal; + } catch (error) { + if (!damagedRecord(error)) throw error; + // Complete native node groups survive this scheduling reset. The + // runner verifies them before acknowledging without provider calls. + } + } } const suffix = `.preserved-${randomUUID()}`; try { @@ -350,6 +364,14 @@ export class EmbeddingJobJournal { private async readFailed(name: string): Promise { const record = (await readEmbeddingJobRecord(this.directory, name)) as FailedUnit; + if ( + record && + typeof record === 'object' && + ((record.version !== undefined && record.version !== 1) || + (record.identity && record.identity !== this.state.identity) || + (record.manifestDigest && record.manifestDigest !== this.state.manifestDigest)) + ) + throw new Error('Embedding failure identity mismatch; recovery refused'); if ( !record || record.version !== 1 || @@ -365,6 +387,30 @@ export class EmbeddingJobJournal { return record; } + /** Constant memory; failed records are the only growing scheduling data. */ + private async verifyFailedRecords(): Promise { + let units = 0; + let nodes = 0; + const entries = await fs.opendir(this.directory); + for await (const entry of entries) { + if (!FAILED.test(entry.name)) continue; + // An active retry owns this unit even if its failure file was already + // removed, or a crash left the previous failed record beside it. + if (this.state.active && entry.name === filename(this.state.active.start.unit)) continue; + const record = await this.readFailed(entry.name); + units++; + nodes += record.intent.nodes; + if ( + record.intent.next.unit > this.state.cursor.unit || + units > this.state.failedUnits || + nodes > this.state.failedNodes + ) + throw new Error('Invalid failed embedding accounting'); + } + if (units !== this.state.failedUnits || nodes !== this.state.failedNodes) + throw new Error('Invalid failed embedding accounting'); + } + async *failed(): AsyncGenerator { const directory = await fs.opendir(this.directory); for await (const entry of directory) { diff --git a/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs b/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs new file mode 100644 index 000000000..05efe9e9c --- /dev/null +++ b/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs @@ -0,0 +1,13 @@ +// Exit after native DDL returns to JS, before the artifact identity is written. +// The parent reopens the real file; no storage or recovery implementation is mocked. +import lbug from '@ladybugdb/core'; +import { VectorStore } from '../../dist/core/lbug/vector-store.js'; + +const query = lbug.Connection.prototype.query; +lbug.Connection.prototype.query = async function (sql, ...args) { + const result = await query.call(this, sql, ...args); + if (sql.startsWith('CREATE NODE TABLE CodeEmbedding')) process.exit(73); + return result; +}; +await VectorStore.open(JSON.parse(process.argv[2])); +throw new Error('Expected interruption during vector schema creation'); diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index e3dcd688a..3a776cd7f 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -9,6 +9,9 @@ import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; import { VectorStore } from '../../src/core/lbug/vector-store.js'; import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; let temporary: TestDBHandle | undefined; afterEach(async () => { @@ -135,6 +138,30 @@ async function fixture(count = 130) { } describe('published graph embedding job controller', () => { + it('resumes an interrupted bootstrap without rescanning the manifest or losing the pending job', async () => { + const f = await fixture(3); + const original = VectorStore.open.bind(VectorStore); + const open = vi.spyOn(VectorStore, 'open').mockImplementationOnce(async (options) => { + await expect( + promisify(execFile)(process.execPath, [ + fileURLToPath(new URL('../helpers/interrupted-vector-bootstrap.mjs', import.meta.url)), + JSON.stringify(options), + ]), + ).rejects.toMatchObject({ code: 73 }); + throw new Error('interrupted bootstrap'); + }); + await expect(f.run()).rejects.toThrow('interrupted bootstrap'); + expect(f.provider).not.toHaveBeenCalled(); + const pending = (await loadMeta(f.root))?.embeddingJob; + expect(pending).toBeDefined(); + f.scan.mockClear(); + open.mockImplementation(original); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.identity).toBe(pending?.identity); + expect(f.scan).not.toHaveBeenCalled(); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + }); it('drains an earlier local runtime before admitting an HTTP job with no model reservation', async () => { const f = await fixture(3); vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'https://embedding.test/v1'); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 881ca92e4..90fd973bf 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -25,6 +25,9 @@ import { publishPreparedAnalysis, } from '../../src/core/analysis-publication.js'; import { ExtensionManager } from '../../src/core/lbug/extension-loader.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; let temporary: TestDBHandle | undefined; let store: VectorStore; @@ -127,6 +130,134 @@ async function runnerFixture(count: number, content = 'const value = "漢😀";' } describe('bounded durable embedding runner', () => { + it('refuses a symlinked native shadow sidecar before opening the artifact', async () => { + const options = await setup(); + await store.close(); + const target = path.join(options.directory, 'unrelated-file'); + await fs.writeFile(target, 'retain this'); + await fs.symlink(target, path.join(options.directory, 'vectors.shadow')); + await expect(VectorStore.open({ ...options, mode: 'resume' })).rejects.toThrow( + 'owned regular file', + ); + expect(await fs.readFile(target, 'utf8')).toBe('retain this'); + }); + it('never bootstraps over an unidentified nonempty native catalog', async () => { + temporary = await createTempDir(); + const options = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'resume' as const, + recoverEmptyBootstrap: true, + }; + const file = path.join(options.directory, 'vectors'); + let handle = await openLbugConnection(lbug, file, { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + const execute = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + await execute('CREATE NODE TABLE PreviousWork (id STRING, PRIMARY KEY(id))'); + await execute("CREATE (:PreviousWork {id:'retain-this'})"); + await closeLbugConnection(handle); + await expect(VectorStore.open(options)).rejects.toThrow(); + handle = await openLbugConnection(lbug, file, { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query('MATCH (n:PreviousWork) RETURN n.id AS id'); + const parts = Array.isArray(result) ? result : [result]; + expect(parts[0].getNextSync()).toEqual({ id: 'retain-this' }); + parts.forEach((part) => part.close()); + } finally { + await closeLbugConnection(handle); + } + }); + it('recovers an interrupted vector bootstrap only with explicit empty-catalog admission', async () => { + temporary = await createTempDir(); + const options = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'create' as const, + }; + await expect( + promisify(execFile)(process.execPath, [ + fileURLToPath(new URL('../helpers/interrupted-vector-bootstrap.mjs', import.meta.url)), + JSON.stringify(options), + ]), + ).rejects.toMatchObject({ code: 73 }); + await expect(VectorStore.open({ ...options, mode: 'resume' })).rejects.toThrow(); + store = await VectorStore.open({ ...options, mode: 'resume', recoverEmptyBootstrap: true }); + await store.beginNode('Function:a:test', 'hash'); + await store.append('Function:a:test', 'hash', [chunk(0)]); + const receipt = await store.finishNode('Function:a:test', 'hash', 1); + await store.checkpointUnit([receipt]); + expect(await store.reusable('Function:a:test', 'hash')).toMatchObject({ + complete: true, + chunks: 1, + }); + }); + + it.each(['checksum', 'missing', 'accounting'])( + 'recovers a damaged failed-unit %s without repeating acknowledged provider requests', + async (damage) => { + const fixture = await runnerFixture(65); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00064'))) + throw new HttpEmbeddingError('bad input', { status: 400 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + expect((await fixture.run()).state).toMatchObject({ + completedUnits: 1, + failedUnits: 1, + completedNodes: 64, + }); + const failedFile = path.join(fixture.options.directory, 'failed-0000000000000001.json'); + if (damage === 'missing') await fs.unlink(failedFile); + else if (damage === 'checksum') await fs.writeFile(failedFile, '{broken'); + else { + const { readEmbeddingJobRecord, writeEmbeddingJobRecord } = + await import('../../src/storage/embedding-job.js'); + const record = (await readEmbeddingJobRecord( + fixture.options.directory, + path.basename(failedFile), + )) as { intent: { nodes: number } }; + await writeEmbeddingJobRecord(fixture.options.directory, path.basename(failedFile), { + ...record, + intent: { ...record.intent, nodes: 2 }, + }); + } + await fixture.reopen(true); + fixture.provider.mockClear(); + fixture.provider.mockImplementation(async (texts) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + const result = await fixture.run(); + expect(result.state).toMatchObject({ + phase: 'indexing', + failedUnits: 0, + reusedNodes: 64, + completedNodes: 1, + chunks: 65, + }); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect( + (await fs.readdir(fixture.options.directory)).some((name) => + name.startsWith('job.json.preserved-'), + ), + ).toBe(true); + }, + ); it('copies compatible complete vectors into a new generation without re-embedding', async () => { const fixture = await runnerFixture(3, 'const value = "漢😀";'.repeat(300)); const first = await fixture.run(); diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index b8706a3f3..30834aa25 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -47,6 +47,49 @@ afterEach(async () => { }); describe('durable embedding scheduling', () => { + it('refuses foreign failed-unit identity instead of resetting another job', async () => { + const journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + const name = 'failed-0000000000000000.json'; + const record = (await readEmbeddingJobRecord(journal.directory, name)) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, name, { ...record, identity: 'f'.repeat(64) }); + const before = await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8'); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toThrow('identity mismatch'); + expect(await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8')).toBe(before); + expect((await fs.readdir(journal.directory)).some((name) => name.includes('.preserved-'))).toBe( + false, + ); + }); + + it('propagates failure-record access errors without resetting valid accounting', async () => { + const journal = await job(); + const before = await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8'); + vi.spyOn(fs, 'opendir').mockRejectedValueOnce( + Object.assign(new Error('access denied'), { code: 'EACCES' }), + ); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toMatchObject({ code: 'EACCES' }); + expect(await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8')).toBe(before); + }); + + it('retains an active retry whose old failed record was already removed', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + await journal.begin(unit(0, 'failed')); + await fs.unlink(path.join(journal.directory, 'failed-0000000000000000.json')); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ failedUnits: 0, active: unit(0, 'failed') }); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + expect(journal.snapshot()).toMatchObject({ reusedUnits: 1, active: null }); + }); it('resumes only the active unit and keeps acknowledged counts', async () => { let journal = await job(); await journal.begin(unit(0)); From e7dc4c121ba90b042d861a945a8bd8f04acb0500 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 16:33:24 +0000 Subject: [PATCH 12/67] fix(resources): verify native pool release Keep failed and concurrent native closes tracked until release succeeds. Refuse another reserved allocation while cleanup is unverified, including failed connection construction. Let graph readers tear down after result-close errors. --- gitnexus/src/core/embeddings/embedding-job.ts | 4 + gitnexus/src/core/lbug/embedding-source.ts | 37 +++- gitnexus/src/core/lbug/lbug-adapter.ts | 7 +- gitnexus/src/core/lbug/lbug-config.ts | 84 ++++++++- gitnexus/src/core/lbug/vector-store.ts | 20 +- gitnexus/src/core/run-analyze.ts | 2 + .../embedding-native-release.test.ts | 139 ++++++++++++++ .../test/unit/lbug-resource-release.test.ts | 174 ++++++++++++++++++ 8 files changed, 448 insertions(+), 19 deletions(-) create mode 100644 gitnexus/test/integration/embedding-native-release.test.ts create mode 100644 gitnexus/test/unit/lbug-resource-release.test.ts diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 931aaada3..2376b0ca8 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -15,6 +15,7 @@ import { import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; +import { drainLbugResourceReleases } from '../lbug/lbug-config.js'; import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; import { embeddingJobIdentity } from './embedding-job-identity.js'; @@ -118,6 +119,9 @@ export async function runPublishedEmbeddingJob( ): Promise { const { resources, signal } = options; signal?.throwIfAborted(); + // A previous failed close must finish before a model or another native pool + // can consume the next invocation's reservation. + await drainLbugResourceReleases(); const root = await embeddingJobDirectory(options.metadataDirectory); let metadata = await options.metadata.loadMetadata(); const graph = metadata?.graphGeneration; diff --git a/gitnexus/src/core/lbug/embedding-source.ts b/gitnexus/src/core/lbug/embedding-source.ts index ba26e7ff5..67b2e15b6 100644 --- a/gitnexus/src/core/lbug/embedding-source.ts +++ b/gitnexus/src/core/lbug/embedding-source.ts @@ -1,6 +1,10 @@ /** Key-only manifest scans and bounded PK fetches against one immutable graph. */ import lbug from '@ladybugdb/core'; -import { openLbugConnection, type LbugConnectionHandle } from './lbug-config.js'; +import { + closeLbugConnection, + openLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; import { EMBEDDABLE_LABELS, LABELS_WITH_EXPORTED, @@ -49,8 +53,12 @@ function optionalString(value: unknown): string | undefined { export class EmbeddingGraphSource implements EmbeddingManifestSource { private active = false; private closed = false; + private closing?: Promise; private readonly fetchStatements = new Map(); - private constructor(private readonly handle: LbugConnectionHandle) {} + private constructor( + private readonly handle: LbugConnectionHandle, + private readonly graphPath: string, + ) {} /** graphPath has already passed the publication layer's ownership validation. */ static async open( @@ -60,6 +68,7 @@ export class EmbeddingGraphSource implements EmbeddingManifestSource { ): Promise { return new EmbeddingGraphSource( await openLbugConnection(lbug, graphPath, { readOnly: true, bufferPoolBytes, threads }), + graphPath, ); } @@ -103,8 +112,11 @@ export class EmbeddingGraphSource implements EmbeddingManifestSource { }; } } finally { - result?.close(); - this.active = false; + try { + result?.close(); + } finally { + this.active = false; + } } } @@ -162,20 +174,25 @@ export class EmbeddingGraphSource implements EmbeddingManifestSource { } return nodes; } finally { - result?.close(); - this.active = false; + try { + result?.close(); + } finally { + this.active = false; + } } } async close(): Promise { - if (this.closed) return; + if (this.closing) return this.closing; if (this.active) throw new Error('Cannot close an active embedding graph read'); this.closed = true; this.fetchStatements.clear(); + this.closing = closeLbugConnection(this.handle, { strict: true, databasePath: this.graphPath }); try { - await this.handle.conn.close(); - } finally { - await this.handle.db.close(); + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; } } } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 83f283f3d..2091e70be 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -59,6 +59,7 @@ import { resolveFtsVersionPair } from './vendored-extension-path.js'; import { classifyDeleteAllError, closeLbugConnection, + drainLbugResourceReleases, HANDLE_RELEASE_PROBE_ATTEMPTS, HANDLE_RELEASE_PROBE_DELAY_MS, isDbBusyError, @@ -815,7 +816,11 @@ export interface LbugInitOptions { } export const initLbug = async (dbPath: string, options: LbugInitOptions = {}) => { - return runWithSessionLock(() => ensureLbugInitialized(dbPath, options)); + return runWithSessionLock(async () => { + if (options.bufferPoolBytes !== undefined || options.threads !== undefined) + await drainLbugResourceReleases(); + return ensureLbugInitialized(dbPath, options); + }); }; /** diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index f4fb73e72..b39e81e59 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -857,6 +857,61 @@ export interface LbugConnectionHandle { conn: lbug.Connection; } +interface NativeRelease { + db: lbug.Database; + conn?: lbug.Connection; + databasePath?: string; + connectionClosed: boolean; + databaseClosed: boolean; + pending?: Promise; +} + +// Only closing/failed handles are held strongly. A rejected close is not proof +// that its native pool was released, including failed connection construction. +const nativeReleases = new Map(); +const reservedHandlePaths = new WeakMap(); + +function releaseNativeResources( + handle: Pick & Partial>, + databasePath?: string, +): Promise { + let release = nativeReleases.get(handle.db); + if (!release) { + release = { ...handle, databasePath, connectionClosed: !handle.conn, databaseClosed: false }; + nativeReleases.set(handle.db, release); + } + if (release.pending) return release.pending; + const state = release; + state.pending = (async () => { + if (!state.connectionClosed) { + if (state.conn) await state.conn.close(); + state.connectionClosed = true; + } + if (!state.databaseClosed) { + await state.db.close(); + state.databaseClosed = true; + } + if ( + process.platform === 'win32' && + state.databasePath && + !(await waitForWindowsHandleRelease(state.databasePath)) + ) + throw new Error('LadybugDB native resource reservation has not been released'); + nativeReleases.delete(state.db); + })().finally(() => { + state.pending = undefined; + }); + return state.pending; +} + +/** Before admitting another analysis phase, retry only previously requested + * closes. Active readers/writers are never closed or interrupted here. */ +export async function drainLbugResourceReleases(): Promise { + for (const release of nativeReleases.values()) { + await releaseNativeResources(release, release.databasePath); + } +} + /** * Return true when the error message indicates that a LadybugDB write * transaction could not proceed due to lock contention — either a file @@ -1158,6 +1213,8 @@ export async function openLbugConnection( (!Number.isSafeInteger(options.threads) || options.threads < 1) ) throw new Error('LadybugDB threads must be a positive integer'); + const reserved = options.bufferPoolBytes !== undefined || options.threads !== undefined; + if (reserved) await drainLbugResourceReleases(); const safePath = toNativeSafePath(databasePath); let db: lbug.Database | undefined; try { @@ -1166,14 +1223,35 @@ export async function openLbugConnection( options.threads === undefined ? new lbugModule.Connection(db) : new lbugModule.Connection(db, options.threads); - return { db, conn }; + const handle = { db, conn }; + if (reserved) reservedHandlePaths.set(handle, safePath); + return handle; } catch (err) { - if (db) await db.close().catch(() => {}); + if (db) { + if (reserved) { + try { + await releaseNativeResources({ db }, safePath); + } catch (releaseError) { + throw new AggregateError( + [err, releaseError], + 'LadybugDB open failed and resource release is unverified', + { cause: err }, + ); + } + } else await db.close().catch(() => {}); + } throw err; } } -export async function closeLbugConnection(handle: LbugConnectionHandle): Promise { +export async function closeLbugConnection( + handle: LbugConnectionHandle, + options: { strict?: boolean; databasePath?: string } = {}, +): Promise { + if (options.strict || reservedHandlePaths.has(handle) || nativeReleases.has(handle.db)) { + await releaseNativeResources(handle, options.databasePath ?? reservedHandlePaths.get(handle)); + return; + } await handle.conn.close().catch(() => {}); await handle.db.close().catch(() => {}); } diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index b82d67efa..d35390e58 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -2,7 +2,11 @@ import lbug, { type LbugValue } from '@ladybugdb/core'; import fs from 'node:fs/promises'; import path from 'node:path'; -import { openLbugConnection, type LbugConnectionHandle } from './lbug-config.js'; +import { + closeLbugConnection, + openLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; import { ExtensionManager } from './extension-loader.js'; import { assertValidGraphNodeId } from '../graph/node-identity.js'; import { embeddingJobDirectory } from '../../storage/embedding-job.js'; @@ -62,6 +66,7 @@ export class VectorStore { private readonly prepared = new Map(); private busy = false; private closed = false; + private closing?: Promise; private poisoned = false; private sealed = false; @@ -632,14 +637,19 @@ export class VectorStore { } async close(): Promise { - if (this.closed) return; + if (this.closing) return this.closing; if (this.busy) throw new Error('Cannot close an active vector operation'); this.closed = true; this.prepared.clear(); + this.closing = closeLbugConnection(this.handle, { + strict: true, + databasePath: path.join(this.options.directory, 'vectors'), + }); try { - await this.handle.conn.close(); - } finally { - await this.handle.db.close(); + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; } } } diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index ebd4dc37f..ceccc7c2d 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -102,6 +102,7 @@ import { DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js'; import { + drainLbugResourceReleases, estimateBufferPool, setBufferPoolSizeHint, resolveNativeSafeStorageDir, @@ -1447,6 +1448,7 @@ export async function runFullAnalysis( } initialiseSearchFTSStemmer(); initialiseSearchFTSCjkSegmentation(); + if (options.resources) await drainLbugResourceReleases(); const contentRetention = contentRetentionFromEnvironment(); // Scope the degraded-parse log throttle to this run (module-level counter // would otherwise stay saturated on a reused process). diff --git a/gitnexus/test/integration/embedding-native-release.test.ts b/gitnexus/test/integration/embedding-native-release.test.ts new file mode 100644 index 000000000..3110ed954 --- /dev/null +++ b/gitnexus/test/integration/embedding-native-release.test.ts @@ -0,0 +1,139 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import { + closeLbugConnection, + drainLbugResourceReleases, + openLbugConnection, +} from '../../src/core/lbug/lbug-config.js'; + +describe('embedding native handle ownership', () => { + let root: string; + const opened: Array<{ close(): Promise }> = []; + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-native-release-')); + }); + afterEach(async () => { + vi.restoreAllMocks(); + for (const handle of opened.splice(0)) await handle.close(); + await drainLbugResourceReleases(); + await fs.rm(root, { recursive: true, force: true }); + }); + + async function open(kind: 'graph' | 'vector') { + if (kind === 'vector') { + const store = await VectorStore.open({ + directory: root, + identity: 'a'.repeat(64), + dimensions: 2, + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + batchBytes: 1024, + mode: 'create', + }); + opened.push(store); + return store; + } + const graphPath = path.join(root, 'graph'); + const writer = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + }); + try { + for (const sql of [ + 'CREATE NODE TABLE File (id STRING, name STRING, filePath STRING, content STRING, PRIMARY KEY(id))', + "CREATE (:File {id:'File:one.ts',name:'one.ts',filePath:'one.ts',content:'export const one = 1;'})", + 'CHECKPOINT', + ]) { + const result = await writer.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } + } finally { + await closeLbugConnection(writer); + } + const source = await EmbeddingGraphSource.open(graphPath, 16 * 1024 * 1024, 1); + opened.push(source); + let count = 0; + for await (const _ of source.scan('File')) count++; + expect(count).toBe(1); + return source; + } + + it.each(['scan', 'fetch'] as const)( + 'allows graph teardown after a %s result fails to close', + async (operation) => { + const source = (await open('graph')) as EmbeddingGraphSource; + const original = lbug.QueryResult.prototype.close; + vi.spyOn(lbug.QueryResult.prototype, 'close').mockImplementationOnce(function ( + this: lbug.QueryResult, + ) { + original.call(this); + throw new Error('result release failed'); + }); + const read = + operation === 'fetch' + ? source.fetch('File', ['File:one.ts']) + : (async () => { + for await (const _ of source.scan('File')) { + /* drain */ + } + })(); + await expect(read).rejects.toThrow('result release failed'); + await expect(source.close()).resolves.toBeUndefined(); + }, + ); + + it.each([ + ['graph', 'connection'], + ['graph', 'database'], + ['vector', 'connection'], + ['vector', 'database'], + ] as const)('retries a failed %s %s close before claiming release', async (kind, part) => { + const handle = await open(kind); + const prototype = part === 'connection' ? lbug.Connection.prototype : lbug.Database.prototype; + const close = vi + .spyOn(prototype, 'close') + .mockRejectedValueOnce(new Error('native release failed')); + await expect(handle.close()).rejects.toThrow('native release failed'); + await handle.close(); + expect(close).toHaveBeenCalledTimes(2); + await handle.close(); + expect(close).toHaveBeenCalledTimes(2); + }); + + it.each(['graph', 'vector'] as const)( + 'keeps concurrent %s close callers pending until native exit', + async (kind) => { + const handle = await open(kind); + const original = lbug.Connection.prototype.close; + let release!: () => void; + const wait = new Promise((resolve) => { + release = resolve; + }); + const close = vi + .spyOn(lbug.Connection.prototype, 'close') + .mockImplementationOnce(async function (this: lbug.Connection) { + await wait; + await original.call(this); + }); + const first = handle.close(); + let finished = false; + const second = handle.close().then(() => { + finished = true; + }); + try { + await Promise.resolve(); + expect(finished).toBe(false); + expect(close).toHaveBeenCalledTimes(1); + } finally { + release(); + await Promise.all([first, second]); + } + expect(finished).toBe(true); + }, + ); +}); diff --git a/gitnexus/test/unit/lbug-resource-release.test.ts b/gitnexus/test/unit/lbug-resource-release.test.ts new file mode 100644 index 000000000..c125974b6 --- /dev/null +++ b/gitnexus/test/unit/lbug-resource-release.test.ts @@ -0,0 +1,174 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type lbug from '@ladybugdb/core'; +import type { LbugConnectionHandle } from '../../src/core/lbug/lbug-config.js'; + +type LbugModule = typeof lbug; + +describe('reserved native resource release', () => { + let native: typeof import('../../src/core/lbug/lbug-config.js'); + beforeEach(async () => { + vi.resetModules(); + native = await import('../../src/core/lbug/lbug-config.js'); + }); + + function handles() { + const conn = { close: vi.fn(async () => {}) }; + const db = { close: vi.fn(async () => {}) }; + return { conn, db, handle: { conn, db } as unknown as LbugConnectionHandle }; + } + + it.each(['conn', 'db'] as const)( + 'propagates a failed %s close and retries its release', + async (part) => { + const h = handles(); + h[part].close.mockRejectedValueOnce(new Error('release failed')); + await expect(native.closeLbugConnection(h.handle, { strict: true })).rejects.toThrow( + 'release failed', + ); + if (part === 'conn') expect(h.db.close).not.toHaveBeenCalled(); + await native.closeLbugConnection(h.handle, { strict: true }); + expect(h[part].close).toHaveBeenCalledTimes(2); + expect(h[part === 'conn' ? 'db' : 'conn'].close).toHaveBeenCalledTimes(1); + }, + ); + + it('coalesces concurrent closes and waits for the physical release', async () => { + const h = handles(); + let release!: () => void; + h.conn.close.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + let completed = false; + const first = native.closeLbugConnection(h.handle, { strict: true }); + const second = native.closeLbugConnection(h.handle, { strict: true }).then(() => { + completed = true; + }); + await Promise.resolve(); + expect(completed).toBe(false); + expect(h.conn.close).toHaveBeenCalledTimes(1); + expect(h.db.close).not.toHaveBeenCalled(); + release(); + await Promise.all([first, second]); + expect(h.db.close).toHaveBeenCalledTimes(1); + }); + + it('blocks another reserved pool until a failed release succeeds', async () => { + const h = handles(); + h.db.close.mockRejectedValue(new Error('pool still allocated')); + await native.closeLbugConnection(h.handle, { strict: true }).catch(() => {}); + const Database = vi.fn(function () { + return handles().db; + }); + const Connection = vi.fn(function () { + return handles().conn; + }); + const module = { Database, Connection } as unknown as LbugModule; + await expect( + native.openLbugConnection(module, '/unused', { bufferPoolBytes: 8 * 1024 * 1024 }), + ).rejects.toThrow('pool still allocated'); + expect(Database).not.toHaveBeenCalled(); + h.db.close.mockResolvedValue(); + const next = await native.openLbugConnection(module, '/unused', { + bufferPoolBytes: 8 * 1024 * 1024, + }); + expect(Database).toHaveBeenCalledTimes(1); + await native.closeLbugConnection(next); + }); + + it('retains a database whose connection constructor and cleanup both failed', async () => { + const h = handles(); + h.db.close.mockRejectedValue(new Error('orphan pool still allocated')); + const Database = vi.fn(function () { + return h.db; + }); + const Connection = vi.fn(function (): typeof h.conn { + throw new Error('connection construction failed'); + }); + const module = { Database, Connection } as unknown as LbugModule; + const allocation = { bufferPoolBytes: 8 * 1024 * 1024 }; + await expect(native.openLbugConnection(module, '/unused', allocation)).rejects.toThrow( + /connection construction|release/, + ); + await expect(native.openLbugConnection(module, '/unused', allocation)).rejects.toThrow( + 'orphan pool still allocated', + ); + expect(Database).toHaveBeenCalledTimes(1); + h.db.close.mockResolvedValue(); + Connection.mockImplementation(function () { + return h.conn; + }); + const next = await native.openLbugConnection(module, '/unused', allocation); + expect(Database).toHaveBeenCalledTimes(2); + await native.closeLbugConnection(next); + }); + + it('preserves best-effort close for callers without a reservation', async () => { + const h = handles(); + h.conn.close.mockRejectedValue(new Error('legacy close failure')); + await native.closeLbugConnection(h.handle); + expect(h.db.close).toHaveBeenCalledTimes(1); + }); + + it('keeps strict release mandatory for a handle opened with a reservation', async () => { + const h = handles(); + const Database = vi.fn(function () { + return h.db; + }); + const Connection = vi.fn(function () { + return h.conn; + }); + const handle = await native.openLbugConnection( + { Database, Connection } as unknown as LbugModule, + '/unused', + { threads: 1, bufferPoolBytes: 8 * 1024 * 1024 }, + ); + h.db.close.mockRejectedValueOnce(new Error('reserved close failed')); + await expect(native.closeLbugConnection(handle)).rejects.toThrow('reserved close failed'); + await native.closeLbugConnection(handle); + expect(h.conn.close).toHaveBeenCalledTimes(1); + expect(h.db.close).toHaveBeenCalledTimes(2); + }); + + it('blocks parser and model admission while a prior native release is still failing', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const { runPublishedEmbeddingJob } = await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + const h = handles(); + h.db.close.mockRejectedValue(new Error('previous allocation is live')); + await native.closeLbugConnection(h.handle, { strict: true }).catch(() => {}); + const onProgress = vi.fn(); + const loadMetadata = vi.fn(); + const provider = vi.fn(); + try { + await expect(runFullAnalysis('/unused', { resources }, { onProgress })).rejects.toThrow( + 'previous allocation is live', + ); + await expect( + runPublishedEmbeddingJob( + { + metadataDirectory: '/unused', + graphPath: '/unused/graph', + resources: resources.embedding, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ), + ).rejects.toThrow('previous allocation is live'); + expect(onProgress).not.toHaveBeenCalled(); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + } finally { + h.db.close.mockResolvedValue(); + await native.drainLbugResourceReleases(); + } + }); +}); From 38982ee86cb302abe9cb44a8b1775c0d23e93030 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 17:00:30 +0000 Subject: [PATCH 13/67] fix(resources): drain graph before phase admission Wait for session work and verify the prior main graph pool has closed before parser or embedding admission. Recheck cancellation after the drain so cancelled jobs do not read or mutate job metadata. --- gitnexus/src/core/embeddings/embedding-job.ts | 5 +- gitnexus/src/core/lbug/lbug-adapter.ts | 9 ++ gitnexus/src/core/run-analyze.ts | 4 +- .../test/unit/lbug-analysis-resources.test.ts | 136 ++++++++++++++++++ 4 files changed, 150 insertions(+), 4 deletions(-) diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 2376b0ca8..4ce17cc4a 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -15,7 +15,7 @@ import { import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; -import { drainLbugResourceReleases } from '../lbug/lbug-config.js'; +import { drainAnalysisNativeResources } from '../lbug/lbug-adapter.js'; import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; import { embeddingJobIdentity } from './embedding-job-identity.js'; @@ -121,7 +121,8 @@ export async function runPublishedEmbeddingJob( signal?.throwIfAborted(); // A previous failed close must finish before a model or another native pool // can consume the next invocation's reservation. - await drainLbugResourceReleases(); + await drainAnalysisNativeResources(); + signal?.throwIfAborted(); const root = await embeddingJobDirectory(options.metadataDirectory); let metadata = await options.metadata.loadMetadata(); const graph = metadata?.graphGeneration; diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 2091e70be..40f7b4933 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -2594,6 +2594,15 @@ export const closeLbug = async (options: { strict?: boolean } = {}): Promise => { + await runWithSessionLock(async () => { + if (conn || db || nativeCloseUnverified) await closeLbug({ strict: true }); + await drainLbugResourceReleases(); + }); +}; + /** * Thrown by {@link wipeLbugDbFiles} when a data-bearing member of the * LadybugDB file family is still present after the bounded diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index ceccc7c2d..fb36a9d6c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -76,6 +76,7 @@ import { forEachQueryRow, executeWithReusedStatement, closeLbug, + drainAnalysisNativeResources, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, @@ -102,7 +103,6 @@ import { DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js'; import { - drainLbugResourceReleases, estimateBufferPool, setBufferPoolSizeHint, resolveNativeSafeStorageDir, @@ -1448,7 +1448,7 @@ export async function runFullAnalysis( } initialiseSearchFTSStemmer(); initialiseSearchFTSCjkSegmentation(); - if (options.resources) await drainLbugResourceReleases(); + if (options.resources) await drainAnalysisNativeResources(); const contentRetention = contentRetentionFromEnvironment(); // Scope the degraded-parse log throttle to this run (module-level counter // would otherwise stay saturated on a reused process). diff --git a/gitnexus/test/unit/lbug-analysis-resources.test.ts b/gitnexus/test/unit/lbug-analysis-resources.test.ts index d9f666ca1..c27fff29e 100644 --- a/gitnexus/test/unit/lbug-analysis-resources.test.ts +++ b/gitnexus/test/unit/lbug-analysis-resources.test.ts @@ -123,4 +123,140 @@ describe('analysis database resource allocations', () => { { bufferPoolBytes: options.bufferPoolBytes, threads: 2 }, ]); }); + + it.each(['parser', 'embedding'] as const)( + 'refuses %s admission while the main pool cannot close', + async (phase) => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const { runPublishedEmbeddingJob } = + await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + await adapter.initLbug(path.join(root, 'lbug'), { + skipFts: true, + bufferPoolBytes: 32 * 1024 * 1024, + threads: 1, + }); + const first = await native.open.mock.results[0].value; + first.db.close.mockRejectedValue(new Error('main pool still allocated')); + await expect(adapter.closeLbug()).rejects.toThrow('main pool still allocated'); + const loadMetadata = vi.fn(async () => null); + const provider = vi.fn(); + const onProgress = vi.fn(); + try { + const run = + phase === 'parser' + ? runFullAnalysis(root, { resources }, { onProgress }) + : runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, 'lbug'), + resources: resources.embedding, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ); + await expect(run).rejects.toThrow('main pool still allocated'); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + expect(onProgress).not.toHaveBeenCalled(); + expect(native.open).toHaveBeenCalledTimes(1); + } finally { + first.db.close.mockResolvedValue(); + } + }, + ); + + it('waits for an active session before releasing its native handle', async () => { + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const wait = new Promise((resolve) => { + release = resolve; + }); + const operation = adapter.withLbugDb( + path.join(root, 'lbug'), + async () => { + started(); + await wait; + }, + { skipFts: true }, + ); + await ready; + const first = await native.open.mock.results[0].value; + let drained = false; + const drain = adapter.drainAnalysisNativeResources().then(() => { + drained = true; + }); + try { + await Promise.resolve(); + expect(drained).toBe(false); + expect(first.conn.close).not.toHaveBeenCalled(); + expect(first.db.close).not.toHaveBeenCalled(); + } finally { + release(); + await Promise.all([operation, drain]); + } + expect(first.conn.close).toHaveBeenCalledTimes(1); + expect(first.db.close).toHaveBeenCalledTimes(1); + }); + + it('does not reopen native storage when an idle phase has nothing to release', async () => { + await adapter.drainAnalysisNativeResources(); + expect(native.open).not.toHaveBeenCalled(); + }); + + it('honors cancellation during admission before reading job metadata', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runPublishedEmbeddingJob } = await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + let started!: () => void; + let release!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const wait = new Promise((resolve) => { + release = resolve; + }); + const operation = adapter.withLbugDb( + path.join(root, 'lbug'), + async () => { + started(); + await wait; + }, + { skipFts: true }, + ); + await ready; + const cancellation = new AbortController(); + const loadMetadata = vi.fn(async () => null); + const provider = vi.fn(); + const outcome = runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, 'lbug'), + resources: resources.embedding, + signal: cancellation.signal, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ).catch((error: unknown) => error); + cancellation.abort(); + release(); + await operation; + expect(await outcome).toMatchObject({ name: 'AbortError' }); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + }); }); From 290ebd9f813ba391bc76cd7212306b8e38b05e6e Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 17:49:48 +0000 Subject: [PATCH 14/67] docs(plans): revise resumable analyze seams --- ...plan-resource-bounded-resumable-analyze.md | 2184 +++++++++++------ 1 file changed, 1492 insertions(+), 692 deletions(-) diff --git a/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md b/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md index 8cdd91d2a..714f12193 100644 --- a/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md +++ b/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md @@ -1,216 +1,280 @@ # GitNexus Engineering Plan — resource-bounded, resumable analyze -> Task: implement [GitNexus #3598](https://github.com/abhigyanpatwari/GitNexus/issues/3598), without performance degradation. -> Depth: Deep; form: full; impact depth: 3. Planning only; implementation requires the LFG plan gate. -> Evidence verified at commit f641342d1fcbfdcb764c5e478db0f69e6cea899a on fix/ci-timing-download-retries. -> Index commit matches HEAD, but analyzer provenance is not established against current source; graph symbol identities also disagree with source. **Stale analyzer provenance — source-weighted limitation.** PDG unavailable. No refresh or build performed. -> Evidence provenance schema 2; global dirty digest 305f776a6b7dae0c761b813156861754467d5410fce87ece8bda66d0ea6d377c; 58 sorted cited paths; exact generated plan path excluded. +> Task: finish [GitNexus #3598](https://github.com/abhigyanpatwari/GitNexus/issues/3598), without performance degradation. +> Deepen cycle 1; full form; impact depth 3. This revision reconciles completed work and changes two integration decisions. It does not claim feature completion. +> Evidence: HEAD 8c6b680f3071025189804ffebac2fd3bc8bf1232 on `feat/resource-bounded-resumable-analyze`. Build-current local index matches HEAD, schema-4 analyzer receipt and working-tree content. Connected MCP graph resources have a native engine mismatch; local backend queries work. PDG has partial/source-offset limitations (§5). +> Evidence provenance schema 2; global dirty digest 0a9c85780067d9afcd0764f307b60891e3cee927ee11eaeb5ec7826d10fd82cd; 86 sorted cited paths; only this generated plan path excluded. No production/test/config edits, builds or test executions in this Deepen pass. ## 1. Objective -Deliver a verified graph and keyword index before optional embeddings finish. Bound retained embedding inputs and queued work by resources, resume paid work after interruption, and report graph and vector readiness independently. Preserve throughput, query latency, embedding text/coverage, and ordinary analysis performance. Memory savings obtained by silently doing less work do not satisfy the request. +Publish a verified graph and keyword index before optional embeddings finish. Bound queued content, expanded chunks and vector work by resources, preserve durable paid work across interruption, and report graph freshness separately from embedding readiness. Preserve existing text/chunk semantics, graph-only/no-op/incremental performance, throughput and query latency at comparable resources. -The acceptance scenarios are: bounded large-corpus input retention; the new graph queryable during and after embedding failure; unchanged-generation retries processing only unfinished units; safe recovery at every publication/write boundary; selective invalidation with reasons; truthful status/query capabilities; independent phase progress; and a short synthetic regression suite plus measured before/after performance. +This revision replaces the original unconditional private-graph immutability/rollback promise with the repository's existing staged versus in-place guarantees. It also replaces in-process native teardown on exit-only CLI/server paths with an actual process-exit boundary. Neither change relaxes the issue's requirement that **after verified graph publication, embedding failure leaves that new graph available**. ## 2. Current Behaviour -- [verified] `queryEmbeddableNodes` reads each eligible label into `allNodes`, including content; the File fallback also materializes its result. `runEmbeddingPipeline` retains nodes, an identity set, hashes, stale IDs, and per-batch expanded texts before sub-batch embedding (`gitnexus/src/core/embeddings/embedding-pipeline.ts:162–277, 606–615, 648–788, 844–865`). Small inference batches therefore do not bound corpus input retention. -- [verified] The pipeline announces an active window before mutation; the default window is 5,000 nodes rounded to batch boundaries. It replaces exact row IDs and removes all rows for nodes with failed chunks (`embedding-pipeline.ts:283–319, 595, 726–729, 775–785, 969–1044`). Preserve these correctness rules. -- [verified] `runFullAnalysisInner` completes graph/FTS work and graph integrity checks before embeddings, but closes and publishes the database and freshness metadata only afterward (`gitnexus/src/core/run-analyze.ts:4759–4782, 4977–5044, 5434–5536`). The shared-store publication wrapper also waits for the inner analysis to return (`run-analyze.ts:1528–1541`). -- [verified] The analyze checkpoint callback drains WAL before recording completed windows. Staged recovery validates a retained staging family and filters unsafe node groups; it is useful existing behavior, not a missing feature (`run-analyze.ts:4995–5041`; `gitnexus/src/storage/embedding-recovery.ts:50–112`; `gitnexus/src/core/embeddings/staged-embedding-recovery.ts:117–185`). -- [verified] Checkpoints record model/dimensions/provider, pending IDs and optional staging/schema identity, but do not contain a complete persisted work manifest keyed to source coverage and analyzer identity (`gitnexus/src/storage/repo-meta.ts:47–53, 502–549`). -- [verified] `getIndexIncompleteReasons` includes embedding pending state, and status requires this entire list to be empty for current metadata. The MCP vector lane probes row counts and queries the graph database without using a completed-vector-generation receipt (`gitnexus/src/core/index-freshness.ts:134–175`; `gitnexus/src/cli/status.ts:321–330`; `gitnexus/src/mcp/local/local-backend.ts:4348–4478`). -- [verified] Analyze maps the embedding percentage to 90–98% overall. SSE correctly waits for the job's terminal status rather than trusting the pipeline's `ready` event (`run-analyze.ts:4980–4988`; `gitnexus/src/server/sse-progress.ts:60–75`). +- [verified] Thirteen commits are present, including the original plan. Twelve provide feasibility evidence and implementation pieces (§7). The common bounded controller still has **no production caller**. Analyze, embeddings sync and `/api/embed` still call `runEmbeddingPipeline` (`gitnexus/src/core/run-analyze.ts:4995`, `gitnexus/src/cli/embeddings-sync.ts:198`, `gitnexus/src/server/api.ts:2289`). The feature is not active end to end. +- [verified] `runFullAnalysis` locks the resolved slot, re-resolves it under the lock, calls its inner coordinator, then handles shared-store publication and releases the lock (`gitnexus/src/core/run-analyze.ts:1450–1575`). The new controller expects its caller to hold that lock. +- [verified] Private incremental writes normally mutate the existing graph. Atomic incremental copying is opt-in; shared commit graphs are immutable. The coordinator currently deletes `existingMeta.graphPath` after private-graph preparation, so it cannot simply accept a newly introduced private generation path without updated routing (`run-analyze.ts:2817–2840`). +- [verified] CLI and server workers set `skipNativeCloseOnExit: true`. The close choice is at `run-analyze.ts:5497–5500`. `closeLbugBeforeExit` checkpoints but leaves native handles alive and defers sidecar housekeeping (`gitnexus/src/core/lbug/lbug-adapter.ts:2521–2532`). Strict native release helpers do not make it safe to override this workaround. +- [verified] CLI skill generation consumes the live `pipelineResult` (`gitnexus/src/cli/analyze.ts:1697` onward). The worker IPC projection deliberately excludes it because serializing the KnowledgeGraph materializes corpus-sized arrays and loses behavior (`gitnexus/src/server/analyze-worker-ipc.ts:1–94`). +- [verified] `runPublishedEmbeddingJob` validates graph receipts and resource reservations, persists a graph-bound job, lazily opens the source/provider, and publishes only after native close. Its reuse source is currently only `metadata.vectorGeneration`; it does not reuse a different interrupted job even though its log promises retained compatible work (`gitnexus/src/core/embeddings/embedding-job.ts:116–330`). +- [verified] The staged publication helper requires an owned staging filename and writes a distinct generation file; it has no explicit in-place mode (`gitnexus/src/core/analysis-publication.ts:116–167,262–303`). The current vector selector checks generation matching but lacks an explicit `incrementalInProgress`/collapse gate (`gitnexus/src/storage/vector-generation.ts:33–81`). +- [verified] Optional resource plumbing, native/session drain, sidecar reservations, readiness consumers and progress types are implemented. The total-budget CLI flag, parser-side model drain, producer/IPC phase details and coherent production activation remain unfinished. Existing focused checks passed; the full candidate suite, whole-command benchmark matrix and final review have not run. ## 3. Relevant Architecture -- [verified] Analyze owns an exclusive index lock; graph writes use the singleton adapter and its checkpoint driver. The read pool opens separate read-only native databases. Its injected in-process database mechanism does not establish cross-process read/write support (`run-analyze.ts:1451–1491`; `gitnexus/src/core/lbug/pool-adapter.ts:658–714, 993–1019`). -- [verified] `streamQuery` explicitly refuses to run with the singleton WAL driver active. Wrapping the current all-content query in this function while embedding writes continue would violate an existing invariant (`gitnexus/src/core/lbug/lbug-adapter.ts:1959–2004`). -- [verified] Vector results already return node IDs and are then hydrated through graph queries. A separate vector database can preserve this lookup shape (`local-backend.ts:4406–4426, 4485–4515`). Native vector-index creation still uses adapter-specific behavior, so changing only the injected query function is insufficient. -- [verified] Shared graphs are immutable, and pending embeddings currently prevent their publication. `resolveGraphPath` recognizes shared-store commit paths; local storage defaults to `lbug` (`gitnexus/src/core/shared-store-analyze.ts:377–407`; `gitnexus/src/storage/shared-store.ts:264–282`). -- [verified] `--memory-budget` is explicitly a main-thread V8 heap limit, with independent worker heaps. Native pools and embedding threads have separate sizing (`gitnexus/src/cli/index.ts:175–179`; `gitnexus/src/cli/analyze.ts:623–660`; `gitnexus/src/core/ingestion/workers/worker-pool.ts:691–702, 1035–1085`; `gitnexus/src/core/lbug/lbug-config.ts:479–513`; `gitnexus/src/core/embeddings/config.ts:74–109`). -- [verified] The three production embedding callers are analyze, `embeddingsSyncCommand`, and the `/api/embed` handler. Their checkpoint orchestration is duplicated (`run-analyze.ts:4977`; `gitnexus/src/cli/embeddings-sync.ts:197–245`; `gitnexus/src/server/api.ts:2278–2337`). +[verified] The existing index lock and native-safe worker lifecycle are the coordination primitives. `runFullAnalysis` already rechecks storage/branch placement after waiting. The auto-sync worker launcher already waits for actual process exit before settlement. Server launch handling checks collapsed graphs before backend publication; that ordering must survive the phase split. + +[inferred] Keeping every private graph physically immutable would add an O(database-size) copy to tiny incremental updates on filesystems without copy-on-write. Logical generation receipts plus the existing in-place dirty-recovery contract avoid that added cost. The active embedding job holds the index lock, so its graph stays stable for its lifetime without requiring immutable files forever. + +[inferred] A thin supervisor is needed only where a graph worker must exit before the embedding phase. It must remain small, count against the total budget and reuse existing CLI/server process machinery. Graph-only and warm no-op invocations must not gain an unconditional extra fork or load model/vector modules. + +[verified] Separate vector artifacts already have bounded native pools and complete per-node receipts. Published artifacts are immutable to writers. A graph-bound scheduling journal and reusable paid vector data have different lifetimes; coupling both to the job identity prevents efficient rebasing after source changes. ## 4. GitNexus Findings -All calls used `repo: "/workspace"`. The overview records HEAD f641342… and a 1.6.12 npm-distribution analyzer running Node 24.19.0. The local runner resolves to `npx gitnexus@latest`; there is no global executable. This session's Node is 22.23.3. Same package version is not proof of matching analyzer bytes. Refresh was skipped because provenance is unverified and the local graph is held by another process; no process was stopped. +Freshness was established with exact equality between the MCP context's schema-4 runner receipt and local `status --json` index receipt, matching HEAD, `up-to-date`, empty incomplete reasons and current content. Local queries use the already built analyzer and isolated navigation store. No refresh/build was necessary in this Deepen pass. Connected `clusters` and `processes` resource reads failed because that server reads storage version 42 while the local index uses 47; local impact summaries supply bounded process context instead. -| Tool and target | Result | Consequence | +| Current local call: `impact(target, upstream, maxDepth:3)` | Result and direct dependents | Treatment | | --- | --- | --- | -| `query(search_query: "embedding pipeline checkpoint staged graph publication runAnalyze")` | Definitions led to embedding and freshness modules; no relevant process chains | Use targeted source reads for execution order. | -| `context(runEmbeddingPipeline, file_path: .../embedding-pipeline.ts)` | Found lines 581–1106; one incoming integration-test file | Production dynamic-import callers are missing from the graph result. | -| `impact(runEmbeddingPipeline, file_path: ..., upstream, maxDepth: 3)` | **UNKNOWN**, “No callers resolved” | Text references establish analyze, sync, and server callers; zero is not an all-clear. | -| `impact(runFullAnalysisInner, upstream, maxDepth: 3)` | **CRITICAL**, 8 impacted; d=1: `runFullAnalysis`, `runNative`, `runViaPool` | Returned UID names a benchmark `gc` function. Source confirms only `runFullAnalysis`; benchmark references are calls to `gc`, not this coordinator. Do not edit those benchmarks. | -| `impact(decideEmbeddingResume, file_path: .../embedding-checkpoint.ts, upstream, maxDepth: 3)` | **HIGH**, d=1: `runFullAnalysisInner`, `embeddingsSyncCommand`, `createServer` | All three production entrypoints need recovery parity tests. | -| `impact(semanticSearch, file_path: .../local-backend.ts, upstream, maxDepth: 3)` | **CRITICAL**, d=1: `_queryImpl`; 11 reported affected processes | Source confirms `_queryImpl` at line 3429. Several deeper process names are inconsistent with source; treat expansion as untrusted. | -| `impact(resolveBudgetHeap, file_path: .../cli/analyze.ts, upstream, maxDepth: 3)` | LOW; d=1: `ensureHeap` | Preserve its heap-only contract and test CLI respawn/watch propagation. | +| `runFullAnalysisInner`, file-scoped | LOW; `runFullAnalysis` | Preserve outer lock, shared-store, registry and cache sequencing. The old plan's benchmark `gc` false positives are retired. | +| `runPublishedEmbeddingJob`, file-scoped | UNKNOWN: “No callers resolved” | Source search confirms no production call; controller tests exist. UNKNOWN is not a safety clearance. | +| `decideEmbeddingResume`, file-scoped | CRITICAL; `runFullAnalysisInner`, `embeddingsSyncCommand`, `/api/embed` handler | Migrate all three callers and test identical force/drop/resume decisions. | +| `semanticSearch`, local-backend file | HIGH; `LocalBackend._queryImpl` | Preserve hybrid dispatch, ranking and graph hydration; test readiness gates. | +| `ensureHeap`, CLI analyze file | LOW; `analyzeCommand`, `watchCommandWithRunnerIdentity` | Preserve explicit heap, environment and watch behavior; avoid nested supervisors. | +| `runWorkerAnalysis`, worker-core file | LOW; message handler in `server/analyze-worker.ts` | Add nonterminal phase messages while retaining one terminal slot and exit-only cleanup. | +| `createLaunchAnalysisWorker`, launcher file | LOW; `createServer` | Delay embedding admission until actual exit; keep collapse/branch/cancel behavior. | -[graph] Deep-mode overview, clusters and processes resources were read; bounded summaries identify Storage, Lbug, Ingestion and Cli areas. They are not exhaustive dependency inventories. `context(runAnalyze, .../run-analyze.ts)` was empty because the real coordinator is `runFullAnalysisInner`; `context(publishSharedGraph, .../storage/shared-store.ts)` was empty because source locates it in `core/shared-store-analyze.ts`. +[graph] Process summaries reach CLI/watch, API and query flows; they are sampled navigation, not exhaustive dependency proofs. [verified] Dynamic-import legacy embedding callers were checked with text search. Previously CRITICAL native-close/session edits remain high-risk seams; rerun their impacts before any further edit rather than carry old caller counts forward. ## 5. Statement-Level PDG Findings -`pdg_query(mode: "controls", target: "gitnexus/src/core/embeddings/embedding-pipeline.ts")` returned **“no PDG layer”**. No CDG/REACHING_DEF edges are claimed. A build-current, isolated index refresh with `--index-only --pdg` belongs to execution, before editing central symbols. +[graph] A PDG layer exists. File-scoped `controls` probes returned truncated sets. Narrowing `impact(mode:"pdg", target:, line:5498)` to the executable graph-close statement returned `pdg-no-block-at-line`, UNKNOWN. The controller's line-211 slice returned 14 statements, was still truncated, and located `loadMetadata` at line 125 while current source has it at line 127. The initial line-5500 probe was a ternary continuation and was corrected before drawing this conclusion. -These are **source-derived ordering constraints**, not reconstructed PDG edges: +These results do **not** establish a complete or correctly aligned handoff slice. No fabricated dependence edges enter the context pack. This is a statement-layer limitation despite current source/build/index provenance; unrelated PDG repair is outside the task. -1. [verified] Validate identities before model requests or deleting rows (`embedding-pipeline.ts:606–615`). Streaming must retain whole-manifest identity validation before mutation, using disk-backed validation when necessary. -2. [verified] Persist the active-window intent before delete/insert; checkpoint durable rows before completed progress (`embedding-pipeline.ts:775–785`; `run-analyze.ts:4995–5041`). -3. [verified] A current content hash on one chunk does not prove the complete chunk set survived. A failed node is removed as a whole (`embedding-pipeline.ts:992–1014`). -4. [verified] The database must be durable and published before advancing metadata; a collapse verdict aborts staged publication (`run-analyze.ts:4763–4774, 5434–5521`). Moving publication earlier must carry these checks with it. -5. [verified] Expanding all chunks of a node batch before inference can exceed an input-byte budget even with a bounded node count (`embedding-pipeline.ts:844–865`). Both content loading and chunk expansion require backpressure. -6. [inferred] Rewriting megabyte-scale `RepoMeta` after every small unit, repeatedly scanning labels for pagination, or rebuilding HNSW per unit could erase the throughput benefit. `saveMeta` serializes the whole metadata document and its legacy mirror (`gitnexus/src/storage/repo-manager.ts:323–333`). Use a compact job journal for frequent updates. +Source-verified constraints that govern implementation: + +1. Checkpoint success is not physical native release. Exit-only graph workers must actually exit before another phase consumes their reserved memory or reopens the graph. +2. Recheck cancellation after waiting for native/session release and before metadata/provider work (`embedding-job.ts:120–127`). +3. In-place mutation invalidates the published graph receipt before the first write; successful verification and durability precede replacement freshness metadata. +4. Graph publication, registry/cache/shared-store finalization and a readable new graph precede embedding admission. Graph-ready is not command completion. +5. Acknowledged units require complete chunk groups and native durability. Model calls cannot be inferred reusable from one row hash. +6. Repeated metadata rewrites, whole-graph copies, job-history scans, repeated HNSW construction and unbounded IPC are performance hazards to exclude and measure. ## 6. Proposed Changes -The following are design decisions to implement and measure, not claims that the new behavior exists. +The following are implementation decisions, not descriptions of already activated behavior. -### A. Separate graph publication from vector work +### A. Logical graph generations with explicit publication modes -In `runFullAnalysisInner` and `runFullAnalysis`, extract a graph-finalization boundary into a new `core/analysis-publication.ts` helper. It owns graph identity/integrity verification, FTS outcome, durable native close, graph/metadata publication, registry registration and parse-cache publication. Execute this boundary before loading the embedding model. Preserve legitimate FTS opt-outs/degraded states; never label unavailable keyword search as available. +Extend `prepareAnalysisPublication`/`publishPreparedAnalysis`, `runFullAnalysisInner`, graph routing and reader gates with explicit **staged** and **in-place** modes. -Use a versioned, compact publication receipt to reconcile a crash between graph-file publication and metadata publication. Record intended generation, source-coverage digest, analyzer identity and schema identity before switching the graph. Recovery either finishes publication of a verified generation or retains the prior one; it must not trust a newer timestamp alone. New generation filenames allow Windows readers to retain old handles without requiring overwrite of an open file. Extend `resolveGraphPath` and storage validation to accept only an owned, validated generation path; legacy local `lbug` and existing shared-store paths remain readable. No cross-slot paths or symlink escapes. +- Staged full builds retain owned generation files and the existing prepared receipt/metadata recovery. Before publication, the previous graph remains available where the existing staged path provides that guarantee. +- Default private incremental writes retain the existing writable graph file and delta write plan. Before mutation, durably mark the attempt/dirty state and invalidate its old generation readiness. After graph/FTS verification, checkpoint and native release, publish a new logical generation bound to covered content, runner/parser/schema identity and that owned path. Do not copy or hash the entire graph file just to advance a generation. +- An interrupted in-place graph write retains the existing dirty/incomplete status and forced-rebuild recovery; it cannot promise the previous graph survived. Do not advertise graph readiness until verification and publication finish. This corrects the original plan's unconditional pre-publication rollback promise. +- A prepared receipt binds its attempt to the expected prior metadata/slot. Revalidate it under the index lock before completion; a later writer superseding the attempt makes it unusable. Do not trust timestamps alone. Pin prepared artifacts against cleanup until completed or safely superseded. +- Shared commit graphs remain physically immutable. After a private slot adopts the new format, retain its mutable graph under an owned noncanonical filename and reuse that same file across logical generations. Retire the canonical legacy name only after native release, using the existing verified publication/retirement mechanism without a full file copy. The physical filename need not encode every new logical generation ID. Legacy no-job slots stay on their current path. Private paths must not be mistaken for shared pointers or stripped unconditionally. Add dirty/collapse gating before any generation-aware vector query. -Create a **separate vector database per generation**, using the existing LadybugDB vector schema and index format, behind a new `core/embeddings/vector-store.ts`. Its writer exclusively owns an unpublished file; graph readers open the already published graph. Once vector coverage is verified and the vector file is checkpointed/closed, atomically publish a small pointer that binds it to the graph generation. Readers never open the active vector writer's file. A graph-preserving embedding failure cannot roll back or mark that graph as an incomplete graph write. +The stable-graph guarantee for embedding is the published receipt **plus the held index lock**. After publication, all embedding failures preserve the new graph/FTS and graph freshness. Preserve existing FTS opt-out/degraded reporting, collapse checks, content retention and cache publication rules. -This avoids a second full-graph copy for every embedding job and avoids changing the native engine's locking model. It adds a second database handle during embeddings and vector queries; account for its pool in the resource budget and measure its cost. Keep graph-only runs on their current fast path. Do not build graph tables/FTS inside the vector-only database. Do not pass its handle through the graph singleton or inadvertently allocate two default-sized native pools. +### B. Native process exit as the CLI/server handoff -Update `publishSharedGraph` to publish verified graph facts independently of vector debt. Keep pending vector work checkout-owned; completed immutable vector artifacts may be reused only under a matching identity. Update store seeding, pointer validation and reclamation together; a pending job pins its graph and vector artifacts. A sibling checkout must never clean or mutate them. +Add a small proposed `core/analysis-phase-supervisor.ts` and adapt the existing CLI respawn and server/auto-sync worker launch paths. Do not introduce a general scheduler or a borrowed-lock/token protocol. -### B. Persist a deterministic, bounded work manifest +1. Use a filesystem-only preflight to keep verified no-op/graph-only cases on their current fast path. A compatible pending job takes embedding-only resume without parsing or rewriting the graph. Recheck all decisions after acquiring the existing slot lock. +2. For actual graph work followed by embeddings on exit-only paths, run the graph phase in its own worker process. Preserve `skipNativeCloseOnExit`; checkpoint and prepare the durable publication receipt. Resolve the final repository name under the existing registry collision rules before CLI operations that need it. Complete live-KnowledgeGraph work in that child, including skill generation, with existing `--index-only` and self-commit protections. Any early registry name registration must still point to an old verified graph or an explicitly incomplete/prepared slot; it cannot publish new graph readiness. Transfer only a bounded allowlisted phase/result receipt, never `pipelineResult` or source maps. +3. Treat the graph-prepared message as nonterminal. Wait for actual child exit. Only then may the supervisor reconcile sidecars, reacquire the ordinary index lock, re-resolve the slot and validate the expected receipt/generation before finishing filesystem publication, registry freshness/shared-store/cache bookkeeping and finalization checks. Revalidate the child's registered name as well as slot ownership; a changed registration must not leave generated CLI context naming a different repository. If another writer superseded the attempt, stop with a clear retry reason before any paid request; do not publish stale metadata. +4. Once the graph is verified readable, refresh server graph readers and emit graph-ready while the command/SSE job stays active. Start the embedding continuation under its own admitted allocations. Keep the supervisor and model sidecar inside the same process-family memory budget. Reuse an existing heap-respawn parent where possible; do not stack supervisors. +5. Cooperative cancellation retains artifacts and waits for native-safe exit. A child that sent prepared/complete but has not exited still owns its resources. Never terminate a native-active thread to enforce a budget. Exactly one final event reflects success, failure or cancellation. Defer the CLI success banner and opt-in self-commit until the final outcome, preserving the pre-run edit-safety snapshot through a bounded projection. Graph-ready must not commit context files or print overall success. -Replace the all-content reader inside `runEmbeddingPipeline` with a new `core/embeddings/embedding-work-manifest.ts` source. For each graph generation, build one durable manifest in label/id order containing identifiers, source bounds, selection information and unit boundaries. Use a key-only ordered scan plus bounded primary-key fetches; evaluate its native query plan first. If ordering cannot stay within the native memory budget, use bounded sorted runs on disk. Never use growing OFFSET pagination or a full content query per page. +For genuinely in-process long-lived callers that already perform safe real close, retain `drainAnalysisNativeResources` and strict physical release accounting. Do not route exit-only callers through that helper as a substitute for process exit. `ensureHeap` sometimes returns without spawning, so ambient heap/opt-out paths need explicit coverage. -Preflight duplicate/invalid IDs across the entire manifest before any delete or provider call. Keep the File fallback only when all eligible symbol labels are empty; a failed label read remains an error. Persist the selection receipt so resume does not re-enumerate completed units. Store identifiers/fingerprints rather than a second copy of raw source in the manifest. Read content from the pinned graph generation, respecting its full/symbol/none retention policy; never substitute changing working-tree text during a job. +### C. Rebase pending paid artifacts without copying the graph -Bound units by input bytes, expected vector bytes and node count. Bound chunk text, update metadata, encoded HTTP bodies, returned vectors and prefetch together; prefetch at most one additional unit and retain one active durable write unit. Large nodes must use bounded content/chunk production preserving current text, overlap, Unicode and line semantics. A node too large for the supported AST working set must take an existing semantically valid bounded path, or fail explicitly before mutation; never silently truncate or claim completion. Inability to handle the synthetic oversized case remains a failed acceptance criterion, not a permitted corpus omission. +Separate the graph-bound **job identity** (manifest, selection, cursor, source/analyzer/schema) from the ownership identity of its **unpublished vector artifact**. Extend `runPublishedEmbeddingJob`, `VectorStore`, `EmbeddingJobJournal`, job metadata and vector publication together. -Move corpus-size identity/hash/pending maps out of the live work queue: use manifest records and indexed vector-store lookups in bounded batches. A small-corpus in-memory optimization is permitted only under the same byte limit and restart semantics. Hash each changed node once. Reuse the existing prepared/batched writes; do not replace them with one new query preparation per row. +An unchanged compatible job resumes its persisted manifest/cursor. A source/graph change creates a fresh graph-bound manifest and journal while adopting the same exclusively owned pending vector artifact through a durable rebase receipt. Validate vector-space and text identities first. For each current node, reuse only a complete matching input/chunk receipt; changed groups are recomputed, deleted/unselected groups are removed before final coverage publication. Do not reinterpret an old journal's completed cursor against a new manifest. -### C. One durable job protocol for all embedding callers +Locate at most one active pending artifact per compatible vector/input space through a directly keyed owned receipt, so returning to a retained space needs no linear job-history scan. Retain at most one previous completed artifact as a read-only fallback while the pending writer accumulates reused/new groups. Repeated source edits reuse that same pending store, so work from the first interrupted attempt survives subsequent rebases without an ever-growing chain of prior jobs or native pools. Published stores are never reopened for writing; the first write after completed publication creates a new pending artifact and performs bounded lookups against the completed source. -Add `core/embeddings/embedding-job.ts` and a filesystem-only `storage/embedding-job.ts` journal. Adapt `runEmbeddingPipeline`, `decideEmbeddingResume`, analyze, embeddings sync and `/api/embed` to that protocol. Preserve existing public function signatures with optional adapter/options additions until all callers migrate; the completed feature must leave no unbounded production entrypoint. +Rebase intent/adoption must be recoverable before mutation. A crash between metadata, ownership receipt and journal changes preserves the old paid artifact, proves one active owner and either finishes or rejects the intended rebase. Incompatible model/dimensions/text identity starts separate owned work with a reason; cleanup must not discard retained paid evidence prematurely. Scheduling/resource changes do not change unit or vector identity. Build HNSW once after current selected coverage is complete, checkpoint and close, then publish the pointer. -Job identity includes graph generation and covered-content digest (dirty worktrees included), source revision, analyzer/parser identity, graph and vector schema fingerprints, provider/model/dimensions, text/chunking identity, and effective text-producing options. Do not include credentials or secret-derived identity data. Batch size, worker count and memory budget are scheduling choices: lowering them must not invalidate already completed vectors or alter persisted unit identity. +### D. Preserve existing vectors without claiming fresh complete coverage -For each unit: (1) fsync active intent; (2) replace all chunk rows for affected nodes; (3) validate complete per-node chunk counts/indices and hashes; (4) drain/checkpoint native writes; (5) atomically persist/fsync the completed cursor and counts; (6) advance. One unit may be active at a time. A crash after durable rows but before cursor publication may replay that one unit idempotently. Previously acknowledged units must not call the provider again. Partial failures remain durable pending records, with no successful coverage claim. +Plain `analyze` must still preserve vectors without requesting new/changed-node embeddings. Keep the legacy no-job fast path lazy and free of migration/model/spool initialization. -Use a compact cursor plus append-only/segmented failed-unit records, not a growing pending-ID array rewritten with full repository metadata each window. Validate journal checksums/version and artifact ownership. On journal loss/corruption, preserve artifacts, derive only provable complete units, and log uncertainty; do not accept isolated matching hashes as completion. +For a previously completed separate vector artifact after a graph update, keep an explicit **preserved historical coverage** reference. It may retain existing semantic-search behavior by hydrating returned IDs against the newly published graph and omitting deleted IDs. Report its source generation and `coverage: preserved` separately from native index availability; never call it complete for the current selection. Do not rewrite all vectors or invoke a provider merely to change this small reference. This is a compatibility exception for a previously completed artifact, not permission to expose an active/failed writer. While a new job is pending/failed, gate the vector lane before model initialization or vector-table access and continue graph/keyword queries. -An unchanged compatible graph takes an embedding-only resume path before parse/resolve/graph-write work. For a source/analyzer/schema change, validate the graph anew but reuse vectors whose complete chunk sets and embedding-input fingerprints still match. Provider/model/dimension or text changes invalidate the affected vector space/input set, not an otherwise valid graph. Explain invalidation before requests. Preserve explicit `--drop-embeddings` and existing `--force` meaning; a forced rebuild may discard scheduling receipts while still preserving provably reusable embeddings as existing recovery tests require. +Legacy monolithic indexes remain readable. A write-triggered migration uses bounded complete-group validation/spill and retains the source until publication succeeds. Preserve recorded checkpoint/model/input provenance; never fill missing historical fields with the current configuration and claim they were verified. Relocation alone does not justify re-embedding a provably compatible group. When an explicit new-generation request lacks the provenance required to establish compatibility, retain the artifact, explain the exact unknown identity before requests and treat those groups as unverified rather than silently reusable. Plain preserve-only analysis still performs zero provider requests in that case. Include the additional first-generation work in migration/request-count evidence. -Legacy monolithic indexes remain readable. Migrate their complete vector groups through the existing bounded spill path only when a vector write is needed; never re-embed merely to relocate storage. Keep the old readable artifact until publication succeeds. Legacy interrupted checkpoints retain their strict validation; do not infer a generation identity they never recorded. Avoid changing the graph schema fingerprint solely for vector storage: `EMBEDDING_SCHEMA` is already excluded (`gitnexus/src/core/lbug/schema.ts:719–729, 819–835`). Validate downgrade behavior explicitly: new metadata must never coexist with an obsolete canonical `lbug` that an older binary could mistake for the current graph. Retain rollback artifacts under generation-specific names and document the required rebuild for binaries that do not support the new format. +Keep explicit force/drop semantics and existing legacy interrupted-checkpoint validation. Do not change graph schema identity solely for vector storage. An older binary must not mistake an obsolete canonical graph for current data; document rebuild/downgrade behavior and preserve rollback artifacts under owned names. -### D. Budget the full resource envelope without changing existing flag semantics +### E. Budget admission and production integration -Add a pure `core/analysis-resource-budget.ts` resolver and a new `--resource-budget ` total-budget option. Preserve `--memory-budget` as the main-heap override. Resolve once at the entrypoint and pass explicit immutable budgets through analyze and embedding adapters; avoid global configuration that leaks between server jobs. The automatic total ceiling uses effective process/container memory with headroom; record the selected allocations. +Expose `--resource-budget ` for the process family; keep `--memory-budget` as the existing main-heap override. Resolve once and pass immutable allocations through CLI, watch, server and sync. Include supervisor RSS, graph/embedding native pools, worker heaps/native allocations, sidecar/model memory, encoded HTTP responses, queues and headroom. Never count thread RSS as separate process RSS. -Account for main heap and native graph working set, worker V8/native allocations, graph and vector native pools, ONNX/sidecar runtime, byte queues, buffers and headroom. Count worker RSS correctly: worker threads share process RSS, while a model subprocess must be counted separately. Reuse `effectiveRamBytes`, CPU limits, existing pool safety floors and worker drain behavior. Release parser capacity before embedding; reuse the released budget instead of reserving all phases simultaneously. Reject impossible explicit combinations before destructive work; do not silently reinterpret an unbounded native-pool override as a compliant hard cap. +Wire `drainEmbeddingSidecarForAnalysis` before parser admission on actual graph work, not on a verified no-op. Preserve physical-release accounting after cancellation and failed closes. Automatic defaults are provisional until matched-resource measurements validate useful concurrency and total memory. Keep bounded manifests/chunk expansion, whole-manifest ID validation, File fallback rules and prepared/batched vector writes already implemented. -Retain existing concurrency when it fits. Use bounded parallel input preparation/inference only when CPU, provider policy and memory allow; serialize database mutations and cursor advancement. Backpressure must pause admission rather than kill a worker inside native code. A configured application budget cannot promise an OS-enforced RSS ceiling for arbitrary native allocations: expose reservations and measured peak, include cgroup stress validation, and fail clearly when minimum viable native/model allocations do not fit. +Activate all three writers together only after the process/publication/rebase/readiness contracts are covered. No production embedding path may retain the corpus-size all-content reader. Update shared-store seeding and cleanup so prepared publications, pending jobs and live readers pin the correct artifacts; siblings cannot mutate or collect them. -### E. Readiness and progress become separate, compatible fields +### F. Truthful phase progress across every boundary -Extend `RepoMeta` with versioned graph/vector generation receipts and a compact job reference. Keep the existing aggregate `incompleteReasons` for compatibility, adding graph-specific completeness/freshness and embedding progress fields. A pending embedding job must not force a graph rebuild or hide the newly current graph. Keep existing vector capability status strings: pending/failed/unverified vector coverage reports `unavailable` with a reason; only a complete artifact reports `vector-index`, or `exact-scan` when complete coverage exists but HNSW is unavailable. Retry exhaustion remains visible as failed/incomplete vector work. - -Gate the MCP vector lane **before** model initialization or vector-table queries. Resolve and pin a matching published vector generation; hydrate IDs from that same graph generation. Generation mismatch returns graph/keyword results with the reason. Preserve concurrent BM25/vector dispatch and exact-scan limits. Update the exported semantic-search path used outside MCP as well as `LocalBackend.semanticSearch`; do not leave a server-only route reading stale in-graph vectors. - -Extend `EmbeddingProgress` and server job progress with graph phase, embedding phase, unit totals/completed/reused/pending/failed, active unit and known node/chunk counts. Define categories without double counting: completed (new) + reused + pending + failed = total; active is a subset of pending. Completion advances only after durable acknowledgement. Retried failures move categories explicitly. Count unknown totals as unknown until the manifest is ready; model download is a separate indicator. CLI prints graph completion and an independent embedding fraction. SSE remains open until job terminality, and the public serializer must explicitly copy only safe numeric/enumerated progress fields (`gitnexus/src/server/ops-snapshot.ts:227–234`). +Carry the existing optional detail fields through actual producers, worker IPC, server jobs, CLI, MCP context, HTTP projection and SSE. Define completed-new + reused + pending + failed = total, with active a subset of pending; unknown totals remain unknown until selection is ready. Advance completion only after durable acknowledgement. Report model download separately. Graph-ready is nonterminal; preserved coverage is not current embedding completion. Public projection copies only enumerated/numeric safe fields, never filesystem paths, ownership tokens or internal identity payloads. Preserve existing response shapes with additive fields and verify all consumers. ## 7. Implementation Sequence -Each step ends with focused checks and a coherent commit. Steps 2–5 provide inert helpers/adapters until the integration switch in step 6; do not expose half-migrated production behavior. +**Already completed at this HEAD — do not repeat:** -1. **Re-anchor and establish the performance baseline.** Use an isolated checkout because this tree has unrelated edits. Build current source with `npm run build`; index that checkout with its explicit local `dist/cli/index.js analyze --index-only --pdg`. Re-run impacts and bounded PDG slices. Investigate graph UID mismatches if still present. Add a short corpus/measurement harness and real-native feasibility tests for separate graph readers, vector writer, generation publication, bounded manifest reads and Windows open handles. Compare the proposed reader/writer path to current behavior before committing to integration. Route back to this plan if the architecture cannot meet the performance requirement. -2. **Add resource contracts and manifest reader.** Tests cover compatibility of heap-only flags, total-budget accounting, stable units, failed reads, duplicate IDs, and maximum live bytes. Keep existing parse algorithms and language-provider hooks unchanged. -3. **Add vector generation storage and durable journal.** Implement exclusive vector ownership, compact job persistence, one-active-unit acknowledgement, complete chunk receipts and safe artifact cleanup. Test all interruption boundaries with real native storage and a deterministic fake provider. -4. **Adapt the embedding pipeline.** Stream content/chunks through bounded units, preserve rejection isolation and all-or-none node recovery, perform bounded cache lookups, and build HNSW once at completion. Verify embedding-input fingerprints and row parity against the baseline. -5. **Add graph/vector publication and reader compatibility.** Implement prepared/publication recovery, safe generation resolution, legacy lookup, query gating and per-artifact pools. Validate both warm and newly started readers while a vector writer is held open. -6. **Integrate all writers and surfaces.** Publish graph/metadata/registry/cache before embedding; add the embedding-only resume branch; use the same job runner for sync and server. Wire shared-store pinning/cleanup, status and phase progress. Enable the new path only when these consumers work together. Keep graph-only and unchanged no-op paths free of model/vector/spool initialization. -7. **Run the regression and performance matrix; document behavior.** Run focused and full checks below. Record raw paired samples and environment, compare total and phase times, peak memory, query latency and retry request counts. Update RUNBOOK and CLI/resource-budget documentation. Regenerate intentional goldens once at the end; do not replace baselines to conceal regressions. -8. **Graph change analysis and review.** Run `detect_changes(scope: "all")` before each commit; partial/truncated results are unresolved. After implementation refresh the graph and invoke `gitnexus-review` on the implementation branch, separately including local changes if any. Do not push or open a PR automatically. +| Commit | Completed result | +| --- | --- | +| `f0f3b9abe` | Original approved plan; superseded only by this Deepen revision. | +| `60ddccefa` | Native storage feasibility and baseline microbenchmarks; whole-command harness still missing. | +| `dbf62eab1` | Pure resource resolver and deterministic bounded manifest. | +| `ec9063f65` | Vector store and durable segmented journal. | +| `ba23148ba` | Bounded runner/text streaming and complete-group reuse. | +| `084c52895` | Staged publication and generation-aware readers; in-place mode still needed. | +| `f2fc492cf` | Published-job controller; no production callers yet. | +| `8bdd572fd` | Optional parser/native resource allocation plumbing. | +| `8dec4f273` | Physical sidecar reservations; parser admission not wired. | +| `700d6aca8` | Readiness/progress consumers; producer/IPC wiring incomplete. | +| `2363a1060` | Interrupted native bootstrap and journal recovery. | +| `f953c569e` | Verified explicit native pool release and retry accounting. | +| `8c6b680f3` | Main graph/session drain before in-process phase admission. | + +Steps 1–5 of the original plan have helper implementations and microbenchmarks. Original step 6 is partial: its consumers/helpers exist, but the new writer path is not activated. Original steps 7–8 remain open. The original end-to-end `measure.mjs` was never created; move that missing work to the first remaining step. + +**Remaining sequence, executable from this HEAD after the revised plan gate:** + +1. Create the paired whole-command benchmark harness and record the unchanged pinned baseline/environment. Use the existing microbenchmarks only as diagnostics. Capture no-op/tiny-incremental and process-startup cost early so the supervisor/publication choices are measured before activation. +2. Extend publication to logical in-place generations and implement the bounded native-exit handoff as inert adapters. Cover crash/slot-race/cancellation and CLI live-graph postprocessing. Keep staged and shared immutability where it already applies. +3. Implement pending-artifact rebase and preserved/legacy coverage policy. Test repeated graph changes between crashes and known-identity zero-request migration. Keep published stores immutable and old artifacts until verified adoption/publication. +4. Complete resource admission, shared-store pinning, progress/IPC and reader coverage gates. Validate all public shapes and no-op laziness. +5. Switch analyze, sync and `/api/embed` coherently to the common bounded runner, including embedding-only resume. Remove all unbounded production callers. Run actual CLI/server phase and reader tests, not only injected controller tests. +6. Run the full regression and performance matrix; tune any repeatable regression before shipment. Update RUNBOOK/CLI and benchmark documentation. Regenerate intentional goldens once at the end; do not replace baselines to hide regressions. +7. Run final graph change analysis and `gitnexus-review` on the implementation branch, plus a separate local surface if needed. Report unresolved findings and platform gaps. No push or PR is authorized by this plan. + +Each implementation commit requires impact before symbol edits, relevant focused checks and complete `detect_changes` before commit. Partial/truncated change analysis is unresolved. Preserve the existing isolated worktree and unrelated user edits. ## 8. Test Strategy -Existing tests to extend (located at the pinned source; representative pipeline, memory and native recovery cases source-read): - -| Existing file | Required regression coverage | +| Test file | Status and required scenarios | | --- | --- | -| `gitnexus/test/unit/embedding-pipeline.test.ts` | Current-hash interrupted windows, window-before-write order, exact-key replacement, input rejection isolation, File fallback; assert bounded content AND expanded text/vector bytes. | -| `gitnexus/test/unit/embedding-checkpoint.test.ts` | Identity changes, legacy kinds, force/drop compatibility, failed retry exhaustion remaining observable. | -| `gitnexus/test/unit/analyze-memory-budget.test.ts` | Existing heap-only CLI contract remains; add total-budget conflicts and watch propagation in a new resource-budget test. | -| `gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts` | Keep durable-group reuse, second-crash recovery, changed/deleted nodes, model mismatch, and explicit drop. Existing suite skips Windows; new publication tests must cover it. | -| `gitnexus/test/integration/embedding-identity-integrity.test.ts` | Identity validity, unique composite keys, complete chunk sets and cold reopen. | -| `gitnexus/test/unit/local-backend-lazy-vector.test.ts` | Pending/mismatched artifacts cause zero vector/model requests; complete artifact keeps lazy loading and ranking. | -| `gitnexus/test/unit/server-sse-payload.test.ts` | Safe additive progress fields; graph-ready is nonterminal; exactly one final event reflecting failures. | -| `gitnexus/test/integration/shared-store-embedding-reuse.test.ts` | A sibling's graph/vector publication and cleanup cannot lose durable work. | +| `gitnexus/test/unit/analysis-publication.test.ts` | **existing** — In-place mutation invalidates generation before writes; no full graph copy/hash on a tiny edit. Staged crash keeps old graph; in-place crash preserves existing dirty-recovery behavior; after publication embedding failures preserve new graph. | +| `gitnexus/test/integration/analyze-phase-publication.test.ts` | **new** — Graph child sends prepared then stays alive: zero provider/open-new-pool calls until exit. Cold/warm MCP and HTTP readers see new graph/FTS while separate vector writer is open. Crash before/after file and metadata publication; competing writer changes slot/generation during handoff: reject stale receipt without paid requests. Registry-name collision/change and --skills/self-commit: context names match final registration; prepared slot never appears graph-ready. | +| `gitnexus/test/unit/analyze-worker-core.test.ts` | **existing** — Graph-prepared is nonterminal; cancellation owns terminal slot; preserve skipNativeCloseOnExit. | +| `gitnexus/test/unit/analyze-worker-ipc.test.ts` | **existing** — Graph getters throw if traversed: bounded phase receipt still serializes; safe counters only. | +| `gitnexus/test/unit/analyze-launch-collapse.test.ts` | **existing** — Collapse rejected before backend publication; no embedding child or model admission. | +| `gitnexus/test/unit/analyze-launch-branch-settle.test.ts` | **existing** — External/shared/branch slot from result is revalidated after child exit. | +| `gitnexus/test/unit/auto-sync-analysis-worker.test.ts` | **existing** — Prepared IPC is not terminal; actual exit gates completion; cancelled native child is not force-killed. | +| `gitnexus/test/integration/embedding-job-controller.test.ts` | **existing** — Unchanged graph skips parsing and complete provider requests; graph changes rebase pending artifact. Two source changes between interruptions retain first-attempt paid groups and previous completed fallback. New model/text/dimensions reject incompatible reuse before requests; smaller resource budget does not invalidate units. | +| `gitnexus/test/integration/embedding-vector-store.test.ts` | **existing** — Current manifest validates complete groups; stale/deleted nodes pruned before publish; published store never mutated. Rebase crash at intent/adoption/checkpoint boundaries preserves owned paid artifact and rejects foreign identity. | +| `gitnexus/test/unit/embedding-job-journal.test.ts` | **existing** — One active unit; corrupted checksum/missing segment preserved and recovered only from complete native groups. | +| `gitnexus/test/integration/embedding-job-restart.test.ts` | **new** — Real child crashes at JS-visible intent/native-write/checkpoint/cursor/HNSW/pointer boundaries; at most active unit replay. ENOSPC, cancellation and repeated interruption keep published graph available and no duplicate/missing chunk rows. | +| `gitnexus/test/integration/embedding-resource-bounds.test.ts` | **new** — Lazy 10k/100k/1m descriptors and oversized Unicode nodes: fixed queue high-water mark, identical text/chunks, no corpus materialization. Smaller real-native case counts parent, graph/embedding child and sidecar peaks without double-counting thread RSS. | +| `gitnexus/test/unit/vector-generation.test.ts` | **existing** — Dirty/collapsed/mismatched graphs gate before native/model access; historical completed artifact is labeled preserved; pending job disables vector lane. | +| `gitnexus/test/unit/local-backend-vector-generation.test.ts` | **existing** — Preserved results hydrate current IDs and reject deleted nodes; active vector writer never opened; query ranking parity. | +| `gitnexus/test/unit/embedding-job-progress.test.ts` | **existing** — Completed+reused+pending+failed=total; active subset of pending; preserved is not current-complete; no false terminal event. | +| `gitnexus/test/unit/server-sse-payload.test.ts` | **existing** — Forward producer/IPC phase details through safe allowlist; graph-ready nonterminal; exactly one final success/failure/cancel event. | +| `gitnexus/test/unit/analyze-memory-budget.test.ts` | **existing** — Heap-only flag retains old meaning, including explicit ambient heap and respawn opt-out. | +| `gitnexus/test/unit/analysis-resource-budget.test.ts` | **existing** — Supervisor+children+native pools+sidecar+queues fit; defaults preserve useful concurrency when feasible. | +| `gitnexus/test/unit/lbug-analysis-resources.test.ts` | **existing** — Failed close remains charged; session drain blocks model/parser; cancellation during drain makes no metadata/provider calls. | +| `gitnexus/test/integration/embedding-native-release.test.ts` | **existing** — Explicit native pools are not reused before verified physical close; failed close retries, result-close failures clear active admission. | +| `gitnexus/test/unit/embedding-sidecar-client.test.ts` | **existing** — Parser phase waits for prior model-child exit; cancellation does not release a physically active request early. | +| `gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts` | **existing** — Keep legacy complete-group reuse, second-crash, changed/deleted nodes, force/drop behavior and known-identity relocation without provider requests. | +| `gitnexus/test/integration/vector-generation-compatibility.test.ts` | **new** — Plain analyze preserves legacy and previously completed vector coverage without automatic requests. Unknown legacy provenance remains unknown; explicit generation logs any unsafe-reuse invalidation; relocation alone never invalidates verified groups. Windows open handles, no VECTOR/FTS, shared sibling and external/branch storage, concurrent writers. | +| `gitnexus/test/integration/shared-store-embedding-reuse.test.ts` | **existing** — Prepared publication/pending vector work pins its artifacts; sibling cannot clean or mutate them. | -New tests, with concrete scenarios: - -- `test/unit/embedding-work-manifest.test.ts`: many small nodes, few huge nodes, Unicode, duplicate IDs on different pages, label-read failure, deleted nodes, low budget, budget change on retry → stable manifest identity, complete text coverage, bounded live bytes, no provider calls before identity preflight. -- `test/unit/analysis-resource-budget.test.ts`: cgroup smaller than host, explicit heap/worker/native overrides, graph/vector phases, tiny budgets → allocations fit or fail before mutation; worker concurrency is preserved when feasible. -- `test/integration/analyze-phase-publication.test.ts`: stop before graph publish → old graph; stop immediately after it → new graph/FTS available; stop between file and metadata publication → recovery reconciles the intended generation without exposing a half graph. Test cold process readers and continuously querying warm readers. -- `test/integration/embedding-job-restart.test.ts`: kill a child before active intent, after intent, during a multi-chunk write, after WAL durability but before completed cursor, after cursor, during HNSW build, and after vector-file publication before pointer publication → at most the active unit replayed, acknowledged units submit zero requests, complete final chunk sets with no duplicates. Repeated crashes, ENOSPC, checkpoint failure, journal corruption and cancellation preserve the published graph. -- `test/integration/embedding-resource-bounds.test.ts`: lazy synthetic 10k/100k/1m-node input descriptors and controlled content payloads → the queue high-water mark is capped independently of corpus size. Do not allocate the complete fixture in the test process. Use a smaller real-native case to verify the reader's allocations and a large-node case to expose expansion, not just node-count bounds. -- `test/integration/vector-generation-compatibility.test.ts`: legacy monolithic index, interrupted legacy checkpoint, local/shared/external/branch storage, Windows open reader, simultaneous sync/analyze, model/dimension change, no FTS/VECTOR extension → correct ownership, graph availability, honest vector mode and stable existing result shapes. - -Execution commands, run under `gitnexus/` (scripts verified in package.json): +The existing tests listed above were located; entries marked **new** are proposed and must be created before naming them in a runnable command. Extend focused commands with those tests once implemented. Existing scripts were verified in `gitnexus/package.json`: ```sh -npm run build -npx tsc --noEmit -npm test -- test/unit/embedding-pipeline.test.ts test/unit/embedding-checkpoint.test.ts test/unit/analyze-memory-budget.test.ts test/unit/local-backend-lazy-vector.test.ts test/unit/server-sse-payload.test.ts -npm run test:integration -- test/integration/analyze-staged-embedding-recovery.test.ts test/integration/embedding-identity-integrity.test.ts test/integration/shared-store-embedding-reuse.test.ts -npm test -node --import tsx bench/embedding-inputs/measure.cjs --check -node --import tsx bench/embedding-identities/measure.cjs --check +cd gitnexus && npm run build +cd gitnexus && npx tsc --noEmit +cd gitnexus && npm run typecheck:tests +cd gitnexus && npm test -- test/unit/analysis-publication.test.ts test/unit/vector-generation.test.ts test/unit/analyze-worker-core.test.ts test/unit/analyze-worker-ipc.test.ts test/unit/embedding-job-journal.test.ts test/unit/analysis-resource-budget.test.ts test/unit/lbug-analysis-resources.test.ts test/unit/server-sse-payload.test.ts +cd gitnexus && npm run test:integration -- test/integration/embedding-job-controller.test.ts test/integration/embedding-vector-store.test.ts test/integration/embedding-native-release.test.ts test/integration/analyze-staged-embedding-recovery.test.ts test/integration/shared-store-embedding-reuse.test.ts +cd gitnexus && npm test +cd gitnexus && node --import tsx bench/embedding-inputs/measure.cjs --check +cd gitnexus && node --import tsx bench/embedding-identities/measure.cjs --check ``` -The build is necessary for shared-package output and real CLI integration tests. Add the newly created tests to the focused commands. Introduce `bench/analyze-resume/measure.mjs` in step 1 with documented arguments; that command is **proposed**, not an existing runnable tool. Do not run it before creation. Planning ran no tests or benchmarks. +Also run ESLint and Prettier checks on changed files and any required package checks. Do not run the candidate performance matrix alongside indexing or the full test suite. Baseline full-suite evidence exists (30,685 passing, 91 skipped, four setup environment failures; the isolated setup rerun passed); it is not a candidate pass. The latest implementation commit has 80 focused passing tests across six suites plus build/type/lint checks. No tests were rerun during this planning-only pass. -Performance protocol: compare the same corpus, source features, Node/native versions, hardware, model/provider, request limits and resource budget, using separate isolated stores. Alternate baseline/candidate order with warmup and at least seven measured pairs; increase samples if noisy. Include cold graph-only analyze, warm no-op, tiny incremental edit, all-cached vectors, fresh embeddings, local inference, a deterministic fast HTTP provider, interrupted resume, and warm/cold graph+keyword+vector queries during embedding. Measure total elapsed time, phase durations, nodes/chunks per second, p50/p95 query latency, CPU, process plus sidecar peak RSS, live queue bytes, native pool reservations, disk bytes and provider requests. Include index/manifest creation, migration and final HNSW time; do not hide costs outside the timer. Preserve ranking and input/output fingerprints. +**Performance protocol:** use baseline `f641342d1fcbfdcb764c5e478db0f69e6cea899a` and candidate in isolated stores, same corpus/features, Node/native versions, provider/model/request limits and comparable resources. Alternate order, warm up, and collect at least seven paired samples; extend if noisy. Include cold graph-only, warm no-op, tiny incremental edit on non-CoW storage, preserved/all-cached vectors, fresh embeddings, local inference, fast deterministic HTTP, interrupted resume, legacy migration and concurrent warm/cold graph/keyword/vector queries. Include manifest creation, process handoff, native reopen, migration and final HNSW inside the timer. -**Performance acceptance:** no repeatable throughput loss or query-latency regression against the pinned baseline at comparable resources. Do not translate the user's requirement into an arbitrary allowed slowdown. Publish raw paired samples and uncertainty; inconclusive measurements are unresolved, not a pass. A stricter optional memory budget naturally changes available resources: compare both implementations under that same budget and separately check default behavior. Any repeatable degradation routes back to tuning/design before shipment. +Measure total/phase time, throughput, query p50/p95, CPU, process-family peak RSS, queue bytes, native reservations, disk I/O and provider request counts. Verify equal text/chunk/output fingerprints and result ranking. Test default allocations separately from a constrained explicit budget, comparing like resources in each case. A cold query unable to read the new graph during embedding is a correctness failure regardless of speed. + +**No performance degradation is an acceptance gate, not an assumed result.** No arbitrary allowed slowdown. Existing native/vector/text microbenchmarks are promising diagnostics but do not establish whole-command non-regression. A seven-pair isolated HTTP response microbenchmark had a candidate/baseline geometric ratio around 1.019 and remains inconclusive; investigate it within the full provider path. Report raw samples and uncertainty, including migration/first-generation costs. Any repeatable regression requires tuning/design work before completion; inconclusive or unavailable measurements remain unresolved. ## 9. Risk and Impact Analysis -- **CRITICAL coordinator / query path, HIGH resume policy:** no code edit is authorized by this planning artifact alone. The LFG proceed decision should explicitly acknowledge these disclosed changes, satisfying GUARDRAILS maintainer sign-off. Recheck impact on the actual implementation checkout. -- **Every reported d=1 dependent is accounted for:** `runFullAnalysis` gets graph-boundary integration; `runNative`/`runViaPool` are rejected graph false positives after source search; analyze/sync/server resume callers get parity tests; `_queryImpl` gets vector gating/ranking tests; `ensureHeap` gets old-flag/new-budget compatibility tests. Missing dynamic-import callers were added through source verification. -- **Publication and native safety:** data-before-metadata ordering, generation pinning, WAL driver shutdown, no native-thread termination, and Windows handles are mandatory. Keep current collapse and scope-incompleteness diagnostics. A generation identifier alone does not certify contents. -- **Storage/migration:** graph schema and vector schema have separate validity; don't force graph rebuilds for vector-only changes. No destructive legacy migration. Keep source artifacts until verified publication. Two native handles and on-disk manifests cost resources; measurements decide acceptability. -- **Partial vectors:** matching row hashes, exhausted retries, `ready`, or successful HNSW creation are not proof of full coverage. New pending/failed jobs cannot appear as complete vector capability. -- **Graph availability:** validate graph queries through both MCP and HTTP, including fresh processes, while the vector write handle remains open. Existing readers continuing to see only the old graph does not satisfy #3598. -- **Scope:** CLI/core/storage/server/tests plus necessary benchmark/documentation files only. No language-resolution changes, dependency upgrades, CI/release/security-configuration changes, or edits to existing dirty files. External storage ownership and public-progress redaction remain intact. +- **CRITICAL resume policy / HIGH query lane:** all current depth-1 dependents in §4 have explicit wiring and tests. Native close/session helpers were previously CRITICAL and require fresh impact before edits. The original proceed choice covered the original design; this structural revision returns to the LFG gate. +- **Native lifetime:** a terminal IPC message is not physical release. Keep the existing exit-only workaround and actual-exit accounting on every path, including cancellation and ambient-heap/no-respawn invocations. +- **Publication:** staged rollback and in-place dirty recovery are different guarantees. The post-publication embedding-failure guarantee applies to both. Never publish a stale handoff receipt after another writer changes the slot/generation. +- **Durable reuse:** job and artifact identity separation must not create two mutable owners or lose the oldest compatible paid groups after repeated rebases. Treat journal corruption/unknown ownership as uncertainty, not an empty job. +- **Compatibility:** preserved historical vectors may remain available under explicit coverage metadata, but active/failed new jobs cannot be advertised as complete. Unknown legacy provenance is never silently upgraded. +- **Performance:** full-DB copying, eager migration, extra no-op forks, unbounded IPC, repeated history scans, constrained default concurrency and extra HTTP processing can violate the user's request. Tests and paired measurements decide acceptance. +- **Scope:** only required CLI/core/storage/server/tests/bench/docs files. No language-provider changes, dependencies, CI/release/security configuration, production credentials, unrelated worktree changes or publishing actions. ## 10. Files Expected to Change -Existing symbol names below were source-verified; all proposed new files/symbols are explicitly new. +The following is the remaining implementation map. Existing helper code should be extended where needed, not re-created; only entries explicitly marked NEW are proposed files. Symbol names/ranges are grounded in the current investigation or unchanged verified source. Re-run impact and locate the precise edit range during work. -| File(s) | Responsibility | +| File | Remaining responsibility | | --- | --- | -| `gitnexus/src/core/run-analyze.ts` | `runFullAnalysis`, `runFullAnalysisInner`: graph publication and embedding-only resume. | -| `gitnexus/src/core/embeddings/embedding-pipeline.ts`, `types.ts`, `config.ts` | `runEmbeddingPipeline`, internal readers, `EmbeddingProgress`, `resolveEmbeddingConfig`: bounded units, adapter dependencies, safe progress. | -| `gitnexus/src/core/embedding-checkpoint.ts` | `decideEmbeddingResume` and checkpoint factories: shared protocol and legacy compatibility. | -| `gitnexus/src/storage/repo-meta.ts`, `embedding-recovery.ts` | `RepoMeta`, recovery reference validation: versioned generation/job receipts. | -| `gitnexus/src/storage/shared-store.ts`, `storage-resolver.ts`; `gitnexus/src/core/shared-store-analyze.ts` | Generation resolution, owned-path validation, `publishSharedGraph`/seeding and pinned-artifact cleanup integration. | -| `gitnexus/src/core/lbug/pool-adapter.ts`, `lbug-adapter.ts`, `lbug-config.ts` | Distinct read/write adapter lifetimes, bounded native reads, explicit per-handle pool reservations; retain legacy callers. | -| `gitnexus/src/core/index-freshness.ts`; `gitnexus/src/cli/status.ts` | Graph freshness independent from embedding debt. | -| `gitnexus/src/cli/index.ts`, `analyze.ts`, `embeddings-sync.ts` | New total-resource flag, existing heap contract, common embedding runner and phase output. | -| `gitnexus/src/core/ingestion/workers/worker-pool.ts` | Pass budget allocation to pool sizing/drain; preserve native-safe lifecycle. | -| `gitnexus/src/mcp/local/local-backend.ts`; `gitnexus/src/server/api.ts` | Matching vector-generation reads, all writer entrypoints and HTTP progress. | -| `gitnexus/src/server/analyze-job.ts`, `ops-snapshot.ts` | Optional typed phase counters and explicit safe serialization. | -| **New:** `gitnexus/src/core/analysis-publication.ts`, `analysis-resource-budget.ts` | Publication protocol and pure budget resolution. | -| **New:** `gitnexus/src/core/embeddings/embedding-work-manifest.ts`, `embedding-job.ts`, `vector-store.ts`; `gitnexus/src/storage/embedding-job.ts` | Bounded work source, execution protocol, vector-only adapter, filesystem-only journal. | -| Existing/new tests in §8; **new** `gitnexus/bench/analyze-resume/measure.mjs` | Crash matrix, performance and bounded-retention evidence. | -| `RUNBOOK.md` and relevant CLI documentation | Resource option, graph/vector status, recovery and measured limits. | +| `gitnexus/src/core/run-analyze.ts` | Integrate logical graph generations, deferred exit-only finalization and embedding-only resume; preserve existing no-op/in-place paths. | +| `gitnexus/src/core/analysis-publication.ts` | Add explicit staged/in-place publication modes, attempt binding and exit-verified recovery without whole-graph copies. | +| `gitnexus/src/core/analysis-phase-supervisor.ts` | NEW proposed filesystem/IPC phase supervisor; use actual child exit, existing lock acquisition and bounded result projection. | +| `gitnexus/src/cli/analyze.ts` | Compose supervision with heap respawn; keep live pipeline-result work in graph child and avoid extra forks for no-op/graph-only runs. | +| `gitnexus/src/cli/index.ts` | Expose --resource-budget and pass immutable allocations without changing --memory-budget. | +| `gitnexus/src/cli/analyze-watch.ts` | Preserve heap flags and safe lifecycle across repeated analyses. | +| `gitnexus/src/server/analyze-worker-core.ts` | Emit graph-prepared handoff separately from terminal completion; retain exit-only native cleanup. | +| `gitnexus/src/server/analyze-worker-ipc.ts` | Allowlist bounded phase receipts/counters; never serialize pipelineResult. | +| `gitnexus/src/server/analyze-launch.ts` | Wait for graph-worker exit, finish publication, refresh graph backend, then continue embeddings; exactly one terminal event. | +| `gitnexus/src/core/auto-sync/analysis-worker-launch.ts` | Preserve exit-based settlement and cooperative cancellation; carry nonterminal graph readiness. | +| `gitnexus/src/core/embeddings/embedding-job.ts` | Separate job identity from unpublished artifact ownership, reuse pending paid groups across rebases, and admit only after native release. | +| `gitnexus/src/core/embeddings/embedding-job-identity.ts` | Keep graph-bound scheduling identity separate from reusable input/vector-space identities. | +| `gitnexus/src/core/embeddings/embedding-job-runner.ts` | Validate/reuse complete groups in the current manifest, preserve durability-before-acknowledgement and bounded queues. | +| `gitnexus/src/core/lbug/vector-store.ts` | Allow an exclusively owned unpublished artifact to outlive one graph-bound journal; retain complete group proofs and immutable published stores. | +| `gitnexus/src/storage/embedding-job-journal.ts` | Crash-safe rebase receipt and fresh manifest/cursor; no in-memory history chain or loss of acknowledged compatible groups. | +| `gitnexus/src/storage/embedding-job.ts` | Owned filesystem artifacts and durable adoption/rebase records. | +| `gitnexus/src/core/embeddings/vector-publication.ts` | Verify current manifest coverage and close durability before publishing the artifact pointer. | +| `gitnexus/src/storage/repo-meta.ts` | Version optional graph mode, job/artifact references, preserved coverage and recorded legacy provenance without inventing missing identities. | +| `gitnexus/src/storage/vector-generation.ts` | Reject dirty/collapsed graphs; distinguish verified-current, preserved historical, legacy and pending states. | +| `gitnexus/src/storage/index-capabilities.ts` | Expose availability separately from current-generation coverage; preserve existing result shapes with additive fields. | +| `gitnexus/src/storage/embedding-job-progress.ts` | Represent preserved coverage and durable pending progress truthfully. | +| `gitnexus/src/core/embedding-checkpoint.ts` | Retain legacy proof/force/drop policy; share one caller decision for new jobs. | +| `gitnexus/src/cli/embeddings-sync.ts` | Route to the common bounded controller after ownership and graph validation. | +| `gitnexus/src/server/api.ts` | Route /api/embed through the controller and carry safe readiness/progress fields. | +| `gitnexus/src/core/shared-store-analyze.ts` | Publish graph facts independently of checkout-owned vector debt; preserve shared immutable graph ownership. | +| `gitnexus/src/storage/shared-store.ts` | Resolve explicit graph mode and pin artifacts referenced by prepared publication, active job or readers. | +| `gitnexus/src/storage/repo-manager.ts` | Preserve owned generation references across metadata and registry updates. | +| `gitnexus/src/core/analysis-resource-budget.ts` | Include supervisor and child process peaks; tune automatic allocations using matched-resource measurements. | +| `gitnexus/src/core/lbug/lbug-adapter.ts` | Keep strict admission for genuinely in-process callers; never use it to override exit-only graph cleanup. | +| `gitnexus/src/core/embeddings/embedding-sidecar-client.ts` | Wire prior-model drain before parser admission, while keeping no-op analysis lazy. | +| `gitnexus/src/core/analysis-progress.ts` | Carry graph and embedding phase counters from actual durable producer events. | +| `gitnexus/src/core/embeddings/types.ts` | Maintain additive safe progress contracts. | +| `gitnexus/src/mcp/local/local-backend.ts` | Gate vector selection before model/native vector work; preserve BM25 parallelism and matching graph hydration. | +| `gitnexus/src/core/embeddings/vector-generation-search.ts` | Serve a completed preserved artifact with explicit coverage state; never serve the active writer. | +| `gitnexus/src/core/index-freshness.ts` | Keep vector debt separate; dirty in-place graph mutation remains unavailable/unverified. | +| `gitnexus/src/cli/status.ts` | Report graph freshness, embedding phase and preserved/current coverage independently. | +| `gitnexus/src/server/ops-snapshot.ts` | Allowlist public progress; omit paths, tokens and internal identities. | +| `gitnexus/src/server/analyze-job.ts` | Represent nonterminal graph readiness and one terminal outcome. | +| `gitnexus/src/server/repo-projection.ts` | Expose additive coverage/readiness fields. | +| `gitnexus/src/mcp/resources.ts` | Expose the same graph/vector state in repository context. | +| `gitnexus/bench/analyze-resume/measure.mjs` | NEW proposed end-to-end paired harness; existing microbenchmarks do not replace it. | +| `gitnexus/bench/analyze-resume/README.md` | Document raw samples, environment, pair ordering, equal-work checks and limits. | +| `RUNBOOK.md` | Document phase recovery, resource flag, preserved coverage, migration and downgrade behavior. | -This is a cross-module change, not a two-file fix. Additional caller wiring discovered by the fresh graph belongs in the execution drift report; architectural expansion returns to the plan gate. +Tests from §8 and existing benchmark artifacts are evidence; proposed harness/tests have absent-path provenance. Cleanup integrations may require adjacent owned-storage code already in the original scope, with impact and targeted verification first; unrelated refactors remain excluded. ## 11. Reusable Implementation Context ```json { "implementation_context": { - "task_summary": "Implement #3598: resource-bounded analyze, early graph publication, durable embedding-only resume, truthful readiness/progress, with no performance degradation.", + "task_summary": "Finish #3598 from completed helper commits: preserve incremental performance, publish verified graph before a bounded durable embedding job, and validate no performance degradation.", "acceptance_criteria": [ "New verified graph and keyword queries work during/after embedding failure.", "Retained content, expanded chunks, vectors and prefetch fit an explicit resource envelope independently of corpus size.", @@ -222,12 +286,12 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc ], "evidence_provenance": { "schema_version": 2, - "head_commit": "f641342d1fcbfdcb764c5e478db0f69e6cea899a", + "head_commit": "8c6b680f3071025189804ffebac2fd3bc8bf1232", "generated_plan_path": "docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md", "global_dirty_digest": { "algorithm": "sha256", "canonicalization": "gitnexus-evidence-provenance-v2 NUL-framed UTF-8 records", - "value": "305f776a6b7dae0c761b813156861754467d5410fce87ece8bda66d0ea6d377c" + "value": "0a9c85780067d9afcd0764f307b60891e3cee927ee11eaeb5ec7826d10fd82cd" }, "cited_path_manifest": [ { @@ -246,22 +310,6 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "worktree_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", "untracked_digest": "absent" }, - { - "path": "CONTRIBUTING.md", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", - "index_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", - "worktree_digest": "sha256:c149877030512c90bc2e8a9557c40d03ec8df0cb62f9a6c28a8e8c4f52de3265", - "untracked_digest": "absent" - }, { "path": "GUARDRAILS.md", "object_kind": { @@ -294,6 +342,22 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "worktree_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", "untracked_digest": "absent" }, + { + "path": "gitnexus/bench/analyze-resume/README.md", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", + "index_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", + "worktree_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", + "untracked_digest": "absent" + }, { "path": "gitnexus/bench/analyze-resume/measure.mjs", "object_kind": { @@ -310,6 +374,102 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "worktree_digest": "absent", "untracked_digest": "absent" }, + { + "path": "gitnexus/bench/analyze-resume/native-baseline.json", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", + "index_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", + "worktree_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/native-primitives.mjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", + "index_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", + "worktree_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/text-streaming-baseline.json", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", + "index_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", + "worktree_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/text-streaming.mjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", + "index_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", + "worktree_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/vector-store-baseline.json", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", + "index_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", + "worktree_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/bench/analyze-resume/vector-store.mjs", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", + "index_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", + "worktree_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", + "untracked_digest": "absent" + }, { "path": "gitnexus/bench/embedding-identities/measure.cjs", "object_kind": { @@ -359,7 +519,7 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "untracked_digest": "absent" }, { - "path": "gitnexus/scripts/bench/fts-evict-reload-rss.mjs", + "path": "gitnexus/src/cli/analyze-watch.ts", "object_kind": { "head": "regular", "index": "regular", @@ -369,9 +529,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", - "index_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", - "worktree_digest": "sha256:0889bfe3b4d83a4751dda0e5db9400140bda04e4a6f0d5fe5291057d67b7085a", + "head_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", + "index_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", + "worktree_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", "untracked_digest": "absent" }, { @@ -433,41 +593,89 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", - "index_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", - "worktree_digest": "sha256:6a08ce4ae25072d8e4b0edcc1f0b3bf1d0a6bdd5fb0d1170c342c3cf9f6be9a8", + "head_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", + "index_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", + "worktree_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/analysis-phase-supervisor.ts", + "object_kind": { + "head": "absent", + "index": "absent", + "worktree": "absent", + "untracked": "absent" + }, + "state": "absent", + "rename_from": null, + "rename_to": null, + "head_digest": "absent", + "index_digest": "absent", + "worktree_digest": "absent", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/analysis-progress.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", + "index_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", + "worktree_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", "untracked_digest": "absent" }, { "path": "gitnexus/src/core/analysis-publication.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", + "index_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", + "worktree_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", "untracked_digest": "absent" }, { "path": "gitnexus/src/core/analysis-resource-budget.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", + "index_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", + "worktree_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", + "index_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", + "worktree_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", "untracked_digest": "absent" }, { @@ -487,7 +695,7 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "untracked_digest": "absent" }, { - "path": "gitnexus/src/core/embeddings/config.ts", + "path": "gitnexus/src/core/embeddings/embedding-job-identity.ts", "object_kind": { "head": "regular", "index": "regular", @@ -497,13 +705,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", - "index_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", - "worktree_digest": "sha256:7ba8ae915a989c161ca262dc3d3501dc0d33b3c6085159a829a08db572256de3", + "head_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", + "index_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", + "worktree_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", "untracked_digest": "absent" }, { - "path": "gitnexus/src/core/embeddings/embedding-identity.ts", + "path": "gitnexus/src/core/embeddings/embedding-job-runner.ts", "object_kind": { "head": "regular", "index": "regular", @@ -513,25 +721,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", - "index_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", - "worktree_digest": "sha256:9c954921fd985fda7cc904f1c1daa64e231ac4cfae8c7148a00deea274841eab", + "head_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", + "index_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", + "worktree_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", "untracked_digest": "absent" }, { "path": "gitnexus/src/core/embeddings/embedding-job.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", + "index_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", + "worktree_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", "untracked_digest": "absent" }, { @@ -545,29 +753,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", - "index_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", - "worktree_digest": "sha256:a680386f29e36826df87f09ddcd34042841e24059e7737806f5b31c944b3706c", + "head_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", + "index_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", + "worktree_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", "untracked_digest": "absent" }, { - "path": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/staged-embedding-recovery.ts", + "path": "gitnexus/src/core/embeddings/embedding-sidecar-client.ts", "object_kind": { "head": "regular", "index": "regular", @@ -577,9 +769,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", - "index_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", - "worktree_digest": "sha256:a5ba08c10aa675cda0163769b996487d7e0d731107da1ecdc892dda78e7a3210", + "head_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", + "index_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", + "worktree_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", + "index_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", + "worktree_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", "untracked_digest": "absent" }, { @@ -593,25 +801,41 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", - "index_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", - "worktree_digest": "sha256:fd2269f5aa16cf736e1f0d0a3d458f1585a1aeaf226a9a5ec236ef4bc2064381", + "head_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", + "index_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", + "worktree_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", "untracked_digest": "absent" }, { - "path": "gitnexus/src/core/embeddings/vector-store.ts", + "path": "gitnexus/src/core/embeddings/vector-generation-search.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", + "index_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", + "worktree_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/embeddings/vector-publication.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", + "index_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", + "worktree_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", "untracked_digest": "absent" }, { @@ -625,25 +849,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", - "index_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", - "worktree_digest": "sha256:600340cfd3edd52d709be4bd94e05150596364ec5962901135db8f8ba399faae", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/ingestion/utils/effective-ram.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", - "index_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", - "worktree_digest": "sha256:01590b626cba51e9d46fb5d0ea595957a7df81084603e8a59f131bf398460ad9", + "head_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", + "index_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", + "worktree_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", "untracked_digest": "absent" }, { @@ -657,9 +865,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", - "index_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", - "worktree_digest": "sha256:4d9d62bb2f66f9377e1bbae9cd0c4719e1b3508beba48913a6317152640ec145", + "head_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", + "index_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", + "worktree_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/core/lbug/embedding-source.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", + "index_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", + "worktree_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", "untracked_digest": "absent" }, { @@ -673,9 +897,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", - "index_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", - "worktree_digest": "sha256:8239a813a3b8b14b29e4b726be20b1466f28c0375d7dda4ba4460d8c518d53d9", + "head_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", + "index_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", + "worktree_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", "untracked_digest": "absent" }, { @@ -689,9 +913,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", - "index_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", - "worktree_digest": "sha256:de2fef03409fd536b2c3afc3153a0cf59c75d1f65db9d4bff9f819caefbe1090", + "head_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", + "index_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", + "worktree_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", "untracked_digest": "absent" }, { @@ -705,13 +929,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", - "index_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", - "worktree_digest": "sha256:baccd1bac94981eac6fd801c52814d5aca64027725bea962606dbe60bd7aa435", + "head_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", + "index_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", + "worktree_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", "untracked_digest": "absent" }, { - "path": "gitnexus/src/core/lbug/schema.ts", + "path": "gitnexus/src/core/lbug/vector-store.ts", "object_kind": { "head": "regular", "index": "regular", @@ -721,9 +945,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", - "index_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", - "worktree_digest": "sha256:289aa7b015e5c9a40fe5a58b7cef06c9bdb2d9165373e3ecae0ffe0e9f46dfdf", + "head_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", + "index_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", + "worktree_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", "untracked_digest": "absent" }, { @@ -737,9 +961,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", - "index_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", - "worktree_digest": "sha256:b755a7ec51a8a487fd4a5dfd1c1e6b8743b95a871f1d2473338b4f8f6ea90dd5", + "head_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", + "index_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", + "worktree_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", "untracked_digest": "absent" }, { @@ -769,9 +993,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", - "index_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", - "worktree_digest": "sha256:8aa5c739428a672d22e39a4fe0fd72be9abd35fc3833f19bf8b2c6e2a2b36dcb", + "head_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", + "index_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", + "worktree_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/mcp/resources.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", + "index_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", + "worktree_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", "untracked_digest": "absent" }, { @@ -785,9 +1025,73 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", - "index_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", - "worktree_digest": "sha256:53da3f880d5e7ad08e32b4560d96ac92efa1d9e4101a98d4d6bf6cb4bcff0576", + "head_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", + "index_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", + "worktree_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/analyze-launch.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", + "index_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", + "worktree_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/analyze-worker-core.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", + "index_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", + "worktree_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/analyze-worker-ipc.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", + "index_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", + "worktree_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/server/analyze-worker.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", + "index_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", + "worktree_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", "untracked_digest": "absent" }, { @@ -801,9 +1105,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", - "index_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", - "worktree_digest": "sha256:0796c4615860ced37d08f1a61e3b4d75891b9ea871990d33a57a311a4e8eb47e", + "head_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", + "index_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", + "worktree_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", "untracked_digest": "absent" }, { @@ -817,13 +1121,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", - "index_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", - "worktree_digest": "sha256:1ce520f885d470dba4dba4f6c3d4cf25af5c9ffd5259f77e9e942baa8b35980b", + "head_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", + "index_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", + "worktree_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", "untracked_digest": "absent" }, { - "path": "gitnexus/src/server/sse-progress.ts", + "path": "gitnexus/src/server/repo-projection.ts", "object_kind": { "head": "regular", "index": "regular", @@ -833,29 +1137,61 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", - "index_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", - "worktree_digest": "sha256:fe36f78d27050ae939cf0f0bd22e16a54c8fa754c7d9414e4b470da2a4585ad8", + "head_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", + "index_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", + "worktree_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/embedding-job-journal.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", + "index_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", + "worktree_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/embedding-job-progress.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", + "index_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", + "worktree_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", "untracked_digest": "absent" }, { "path": "gitnexus/src/storage/embedding-job.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", + "index_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", + "worktree_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", "untracked_digest": "absent" }, { - "path": "gitnexus/src/storage/embedding-recovery.ts", + "path": "gitnexus/src/storage/index-capabilities.ts", "object_kind": { "head": "regular", "index": "regular", @@ -865,9 +1201,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", - "index_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", - "worktree_digest": "sha256:b01ff4af596ad37f537b11ef9c54cc8858f79408cc061f15649b2336c1e84ce6", + "head_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", + "index_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", + "worktree_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/src/storage/index-lock.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", + "index_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", + "worktree_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", "untracked_digest": "absent" }, { @@ -881,9 +1233,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", - "index_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", - "worktree_digest": "sha256:ae9a7ea8f2e3d8e630494310176f07e815aab0042ddf2535875c6074c1d1da29", + "head_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", + "index_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", + "worktree_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", "untracked_digest": "absent" }, { @@ -897,9 +1249,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", - "index_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", - "worktree_digest": "sha256:1fd29de696da92b0379341d1825de7b946c20ed4e20b341f2368f19213b46e78", + "head_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", + "index_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", + "worktree_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", "untracked_digest": "absent" }, { @@ -913,13 +1265,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", - "index_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", - "worktree_digest": "sha256:509a05c5573c463337c44af5a5c59207e1f88a6a6154ef30df87f0398ac7b3c1", + "head_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", + "index_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", + "worktree_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", "untracked_digest": "absent" }, { - "path": "gitnexus/src/storage/storage-resolver.ts", + "path": "gitnexus/src/storage/vector-generation.ts", "object_kind": { "head": "regular", "index": "regular", @@ -929,9 +1281,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", - "index_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", - "worktree_digest": "sha256:7e2395f0cfa16b135f18ae44b4a2809b2ccd2d6006ca3f48567bf2fbe57f8757", + "head_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", + "index_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", + "worktree_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", "untracked_digest": "absent" }, { @@ -967,7 +1319,7 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "untracked_digest": "absent" }, { - "path": "gitnexus/test/integration/embedding-identity-integrity.test.ts", + "path": "gitnexus/test/integration/embedding-job-controller.test.ts", "object_kind": { "head": "regular", "index": "regular", @@ -977,9 +1329,9 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", - "index_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", - "worktree_digest": "sha256:f706e6341466878b3e4708cd9e6de7924721ed34807dd3c2c6d23a315785a2a1", + "head_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", + "index_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", + "worktree_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", "untracked_digest": "absent" }, { @@ -998,6 +1350,22 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "worktree_digest": "absent", "untracked_digest": "absent" }, + { + "path": "gitnexus/test/integration/embedding-native-release.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", + "index_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", + "worktree_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", + "untracked_digest": "absent" + }, { "path": "gitnexus/test/integration/embedding-resource-bounds.test.ts", "object_kind": { @@ -1014,6 +1382,22 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "worktree_digest": "absent", "untracked_digest": "absent" }, + { + "path": "gitnexus/test/integration/embedding-vector-store.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", + "index_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", + "worktree_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", + "untracked_digest": "absent" + }, { "path": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", "object_kind": { @@ -1047,19 +1431,67 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "untracked_digest": "absent" }, { - "path": "gitnexus/test/unit/analysis-resource-budget.test.ts", + "path": "gitnexus/test/unit/analysis-publication.test.ts", "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", + "head": "regular", + "index": "regular", + "worktree": "regular", "untracked": "absent" }, - "state": "absent", + "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", + "head_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", + "index_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", + "worktree_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/analysis-resource-budget.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", + "index_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", + "worktree_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/analyze-launch-branch-settle.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", + "index_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", + "worktree_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/analyze-launch-collapse.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", + "index_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", + "worktree_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", "untracked_digest": "absent" }, { @@ -1079,7 +1511,7 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "untracked_digest": "absent" }, { - "path": "gitnexus/test/unit/embedding-checkpoint.test.ts", + "path": "gitnexus/test/unit/analyze-worker-core.test.ts", "object_kind": { "head": "regular", "index": "regular", @@ -1089,13 +1521,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", - "index_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", - "worktree_digest": "sha256:850c9b5e3e3ce8eedb02a1a41d2665d1f671a969dd4424df8d0ad7c05e47ee0f", + "head_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", + "index_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", + "worktree_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", "untracked_digest": "absent" }, { - "path": "gitnexus/test/unit/embedding-pipeline.test.ts", + "path": "gitnexus/test/unit/analyze-worker-ipc.test.ts", "object_kind": { "head": "regular", "index": "regular", @@ -1105,29 +1537,13 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", - "index_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", - "worktree_digest": "sha256:eb8cd73dffebecbcb68033ed1b7b884ec0fa077f8023ca83066a50d83a053955", + "head_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", + "index_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", + "worktree_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", "untracked_digest": "absent" }, { - "path": "gitnexus/test/unit/embedding-work-manifest.test.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/local-backend-lazy-vector.test.ts", + "path": "gitnexus/test/unit/auto-sync-analysis-worker.test.ts", "object_kind": { "head": "regular", "index": "regular", @@ -1137,9 +1553,89 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", - "index_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", - "worktree_digest": "sha256:0a371ca3a3b81b00f67144cc768a4d3ed0be9a80e8a3812409fcbd61e9ee1689", + "head_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", + "index_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", + "worktree_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-job-journal.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", + "index_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", + "worktree_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-job-progress.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", + "index_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", + "worktree_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/embedding-sidecar-client.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", + "index_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", + "worktree_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/lbug-analysis-resources.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", + "index_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", + "worktree_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/local-backend-vector-generation.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", + "index_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", + "worktree_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", "untracked_digest": "absent" }, { @@ -1153,9 +1649,25 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "state": "clean", "rename_from": null, "rename_to": null, - "head_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", - "index_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", - "worktree_digest": "sha256:985f982509a800162beb58b3fc274a9cf432aaf8b1bc3a49611a95f8b24834fe", + "head_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", + "index_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", + "worktree_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", + "untracked_digest": "absent" + }, + { + "path": "gitnexus/test/unit/vector-generation.test.ts", + "object_kind": { + "head": "regular", + "index": "regular", + "worktree": "regular", + "untracked": "absent" + }, + "state": "clean", + "rename_from": null, + "rename_to": null, + "head_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", + "index_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", + "worktree_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", "untracked_digest": "absent" } ] @@ -1164,36 +1676,36 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc { "symbol": "runFullAnalysisInner", "file": "gitnexus/src/core/run-analyze.ts", - "lines": "1566–5794", - "role": "Graph and embedding phase orchestration; CRITICAL graph impact, IDs unreliable", + "lines": "1580,2817-2840,5497-5500,5545-5580", + "role": "Graph finalization; retain in-place vs staged semantics and defer exit-only publication", "source_verified": true }, { - "symbol": "runEmbeddingPipeline", - "file": "gitnexus/src/core/embeddings/embedding-pipeline.ts", - "lines": "581–1106", - "role": "Bounded source, checkpoint windows, complete chunk persistence; UNKNOWN upstream", + "symbol": "runPublishedEmbeddingJob", + "file": "gitnexus/src/core/embeddings/embedding-job.ts", + "lines": "116-330", + "role": "Bounded controller, verified graph admission, pending-artifact rebase", "source_verified": true }, { "symbol": "decideEmbeddingResume", "file": "gitnexus/src/core/embedding-checkpoint.ts", - "lines": "190–290", - "role": "Resume identity/flags and common caller policy; HIGH impact", + "lines": "190-290", + "role": "Existing force/drop/legacy compatibility across three production callers", "source_verified": true }, { "symbol": "LocalBackend.semanticSearch", "file": "gitnexus/src/mcp/local/local-backend.ts", - "lines": "4348–4528", - "role": "Generation-ready vector lane and graph hydration; CRITICAL impact", + "lines": "generation selection and graph hydration; re-locate exact edit range before change", + "role": "Readiness and preserved coverage gating", "source_verified": true }, { - "symbol": "resolveBudgetHeap", + "symbol": "ensureHeap", "file": "gitnexus/src/cli/analyze.ts", - "lines": "623–660", - "role": "Preserve heap-only flag contract; LOW impact", + "lines": "682-764", + "role": "Existing conditional respawn; supervisor cannot assume a parent already exists", "source_verified": true } ], @@ -1201,130 +1713,142 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc { "symbol": "runFullAnalysis", "relationship": "CALLS runFullAnalysisInner", - "relevance": "exclusive index lock and shared-store publication" + "relevance": "Lock, shared-store, cache and registry boundary" }, { - "symbol": "queryEmbeddableNodes", - "relationship": "called by runEmbeddingPipeline", - "relevance": "all-content materialization" + "symbol": "analyzeCommandImpl", + "relationship": "CALLS runFullAnalysis", + "relevance": "CLI postprocessing consumes live graph" }, { - "symbol": "contentHashForNode", - "relationship": "called by pipeline", - "relevance": "stable text identity; no semantic shortcuts" + "symbol": "runWorkerAnalysis", + "relationship": "CALLS runFullAnalysis", + "relevance": "skipNativeCloseOnExit plus terminal projection" }, { - "symbol": "batchInsertEmbeddings", - "relationship": "called by pipeline", - "relevance": "validated exact-key replacement" + "symbol": "createLaunchAnalysisWorker", + "relationship": "parent of graph worker", + "relevance": "Actual exit before continuation and backend publication" }, { - "symbol": "buildVectorIndex", - "relationship": "called by pipeline", - "relevance": "adapter-specific HNSW operation; publish once complete" + "symbol": "projectAnalyzeResultForIpc", + "relationship": "projection", + "relevance": "Bounded scalar allowlist" + }, + { + "symbol": "watchCommandWithRunnerIdentity", + "relationship": "CALLS ensureHeap/runFullAnalysis", + "relevance": "Repeated process lifecycle" }, { "symbol": "embeddingsSyncCommand", - "relationship": "calls pipeline and resume policy", - "relevance": "production parity" + "relationship": "CALLS decideEmbeddingResume", + "relevance": "Production writer parity" }, { - "symbol": "createServer", - "relationship": "contains /api/embed caller", - "relevance": "production parity and safe API progress" + "symbol": "createServer / POST /api/embed", + "relationship": "CALLS decideEmbeddingResume", + "relevance": "Production writer parity" }, { - "symbol": "_queryImpl", - "relationship": "calls LocalBackend.semanticSearch", - "relevance": "concurrent keyword/vector lanes" + "symbol": "LocalBackend._queryImpl", + "relationship": "CALLS semanticSearch", + "relevance": "Preserve hybrid search concurrency/ranking" }, { - "symbol": "getIndexIncompleteReasons", - "relationship": "status consumer", - "relevance": "aggregate legacy incomplete reasons" + "symbol": "prepareAnalysisPublication / publishPreparedAnalysis", + "relationship": "publication helper", + "relevance": "Extend staged-only receipt to explicit in-place mode" + }, + { + "symbol": "selectVectorGeneration", + "relationship": "read gate", + "relevance": "Dirty graph, pending writer and preserved coverage" + }, + { + "symbol": "VectorStore", + "relationship": "native storage", + "relevance": "Stable unpublished artifact plus complete group receipts" + }, + { + "symbol": "EmbeddingJobJournal", + "relationship": "durability", + "relevance": "Graph-specific manifest/cursor and crash-safe rebase" + }, + { + "symbol": "runEmbeddingJob", + "relationship": "runner", + "relevance": "Bounded complete-unit processing" + }, + { + "symbol": "drainAnalysisNativeResources", + "relationship": "resource admission", + "relevance": "Main singleton and tracked explicit pool release" + }, + { + "symbol": "drainEmbeddingSidecarForAnalysis", + "relationship": "resource admission", + "relevance": "Release prior runtime before parser phase" }, { "symbol": "publishSharedGraph", - "relationship": "called by runFullAnalysis", - "relevance": "immutable store publication and pending vector ownership" + "relationship": "shared storage", + "relevance": "Immutable shared graph separate from checkout vector work" }, { "symbol": "resolveGraphPath", - "relationship": "storage path resolution", - "relevance": "safe generation/legacy compatibility" + "relationship": "routing", + "relevance": "Owned canonical/generation/shared paths" }, { - "symbol": "resolveEmbeddingRecovery", - "relationship": "legacy checkpoint validation", - "relevance": "owned staging file families" + "symbol": "getGraphIncompleteReasons", + "relationship": "status", + "relevance": "Graph debt separate from vectors" }, { - "symbol": "recoverStagedEmbeddings", - "relationship": "legacy recovery", - "relevance": "reuse complete groups with bounded vector spill" - }, - { - "symbol": "saveMeta", - "relationship": "metadata publication", - "relevance": "full metadata serialized; avoid per-unit rewrites" - }, - { - "symbol": "streamQuery", - "relationship": "native read adapter", - "relevance": "must not run with singleton WAL driver" - }, - { - "symbol": "ensureHeap", - "relationship": "calls resolveBudgetHeap", - "relevance": "CLI respawn compatibility" - }, - { - "symbol": "createWorkerPool", - "relationship": "resource consumer", - "relevance": "worker caps and native-safe lifecycle" - }, - { - "symbol": "estimateBufferPool", - "relationship": "native resource sizing", - "relevance": "preserve page-size/COPY floors" - }, - { - "symbol": "publicOpsProgress", - "relationship": "SSE serializer", - "relevance": "explicit safe optional fields" - }, - { - "symbol": "resolveEmbeddingIdentity", - "relationship": "resume identity", - "relevance": "secret-free provider/model/dimensions" + "symbol": "runEmbeddingPipeline", + "relationship": "legacy production path", + "relevance": "Three callers remain; must be removed from unbounded production use at activation" } ], "execution_path": [ - "Current: lock -> graph parse/resolve/write/FTS/verify -> embedding windows -> final close/swap/meta/registry/cache -> shared publication.", - "Proposed: lock -> verify compatible published graph/job for fast resume; otherwise build/verify/publish graph and metadata -> immutable graph reader + exclusive unpublished vector writer -> intent/write/checkpoint/ack units -> verify complete coverage/HNSW -> publish matching vector pointer." + "Lock and revalidate target; take existing no-op or embedding-only resume when valid.", + "If graph work is required, drain prior runtime and run bounded graph phase with its existing in-place/staged choice.", + "Checkpoint and prepare receipt; finish live-graph CLI work in graph child; on exit-only paths observe actual child exit.", + "Reacquire existing slot lock, revalidate attempt/slot/metadata, finalize sidecars and publication, registry/shared-store/cache; graph-ready is nonterminal.", + "Reuse/rebase one owned pending vector artifact; persist graph-specific manifest/journal; admit separate vector writer/model within budget.", + "Verify current selected coverage, checkpoint/close, publish vector receipt; report exactly one final outcome." ], "pdg_constraints": [], - "pdg_availability": "No PDG layer. Source-derived constraints in section 5; no reconstructed PDG edges.", + "pdg_availability": "Layer exists. Bounded controls/impact probes are partial and source-offset inconsistent; no trusted complete slice for native handoff. Do not infer edges from source.", + "source_constraints": [ + "Exit-only CHECKPOINT does not release graph/native handles.", + "In-place mutation must invalidate the published graph receipt before the first write.", + "One index lock pins a graph during the active embedding job; a handoff gap requires revalidation.", + "Only complete per-node chunk receipts qualify for reuse; durability precedes cursor acknowledgement.", + "No model initialization or vector table access before readiness gate; no-op stays lazy.", + "Final registry name used by graph-dependent CLI outputs must match post-exit publication; early registration cannot advertise new graph readiness." + ], "architectural_patterns": [ { - "pattern": "One canonical writer lock", - "example_location": "gitnexus/src/core/run-analyze.ts:1451", - "usage_guidance": "Recheck storage/source identity under lock; never let sync/analyze/server race publication." + "pattern": "Owned filesystem receipts and safe publication", + "example_location": "gitnexus/src/core/analysis-publication.ts", + "usage_guidance": "Extend explicit publication modes; do not weaken ownership or checksum checks." }, { - "pattern": "Immutable published graph, separately owned unpublished vectors", - "example_location": "gitnexus/src/core/shared-store-analyze.ts:377", - "usage_guidance": "Extend existing immutable shared-store behavior to phase publication; budget separate native handles." + "pattern": "Bounded IPC projection", + "example_location": "gitnexus/src/server/analyze-worker-ipc.ts:projectAnalyzeResultForIpc", + "usage_guidance": "Keep KnowledgeGraph in child; allowlist only phase receipt and scalar counters." }, { - "pattern": "Exact-key replacement and complete node groups", - "example_location": "gitnexus/src/core/embeddings/embedding-pipeline.ts:283", - "usage_guidance": "Retain idempotent replacement and chunk completeness checks." + "pattern": "Actual exit before settlement", + "example_location": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", + "usage_guidance": "Reuse exit/cancellation discipline; never force-kill native work." }, { - "pattern": "Lazy vector lane", - "example_location": "gitnexus/src/mcp/local/local-backend.ts:4359", - "usage_guidance": "Gate completeness before model import; keep keyword/graph lane available." + "pattern": "Exclusive index lock and target revalidation", + "example_location": "gitnexus/src/core/run-analyze.ts:runFullAnalysis", + "usage_guidance": "At handoff reacquire existing lock and reject replaced receipt/slot/generation." } ], "files_to_modify": [ @@ -1334,365 +1858,641 @@ This is a cross-module change, not a two-file fix. Additional caller wiring disc "runFullAnalysis", "runFullAnalysisInner" ], - "intended_change": "Publish verified graph before embedding; compatible embedding-only resume" + "intended_change": "Integrate logical graph generations, deferred exit-only finalization and embedding-only resume; preserve existing no-op/in-place paths." }, { - "file": "gitnexus/src/core/embeddings/embedding-pipeline.ts", + "file": "gitnexus/src/core/analysis-publication.ts", "symbols": [ - "runEmbeddingPipeline", - "queryEmbeddableNodes", - "semanticSearch" + "prepareAnalysisPublication", + "publishPreparedAnalysis" ], - "intended_change": "Bound source/chunk/vector work; adapter-backed vector reads and writes" + "intended_change": "Add explicit staged/in-place publication modes, attempt binding and exit-verified recovery without whole-graph copies." }, { - "file": "gitnexus/src/core/embeddings/types.ts", - "symbols": [ - "EmbeddingProgress" - ], - "intended_change": "Add optional durable phase counters" - }, - { - "file": "gitnexus/src/core/embeddings/config.ts", - "symbols": [ - "resolveEmbeddingConfig" - ], - "intended_change": "Pass bounded scheduling options without changing text identity" - }, - { - "file": "gitnexus/src/core/embedding-checkpoint.ts", - "symbols": [ - "decideEmbeddingResume", - "mintInterruptedCheckpoint", - "mintPartialCheckpoint" - ], - "intended_change": "Shared recovery and legacy migration" - }, - { - "file": "gitnexus/src/storage/repo-meta.ts", - "symbols": [ - "RepoMeta", - "EmbeddingRecoveryReference" - ], - "intended_change": "Versioned generation receipts and job reference" - }, - { - "file": "gitnexus/src/storage/embedding-recovery.ts", - "symbols": [ - "resolveEmbeddingRecovery" - ], - "intended_change": "Validate legacy/new artifact ownership" - }, - { - "file": "gitnexus/src/storage/shared-store.ts", - "symbols": [ - "resolveGraphPath" - ], - "intended_change": "Resolve owned immutable generation or legacy path" - }, - { - "file": "gitnexus/src/storage/storage-resolver.ts", - "symbols": [ - "inspectCodeIndexDB" - ], - "intended_change": "Validate new generation resolution" - }, - { - "file": "gitnexus/src/core/shared-store-analyze.ts", - "symbols": [ - "publishSharedGraph" - ], - "intended_change": "Separate shared graph validity from checkout vector debt; seed/pin/reclaim consistently" - }, - { - "file": "gitnexus/src/core/lbug/pool-adapter.ts", - "symbols": [ - "initLbug" - ], - "intended_change": "Generation-pinned graph/vector readers with explicit native reservations" - }, - { - "file": "gitnexus/src/core/lbug/lbug-adapter.ts", - "symbols": [ - "streamQuery" - ], - "intended_change": "Bounded read adapter; preserve checkpoint-driver exclusion" - }, - { - "file": "gitnexus/src/core/lbug/lbug-config.ts", - "symbols": [ - "estimateBufferPool", - "setBufferPoolSizeHint" - ], - "intended_change": "Budget all concurrent native pools" - }, - { - "file": "gitnexus/src/core/index-freshness.ts", - "symbols": [ - "getIndexIncompleteReasons" - ], - "intended_change": "Separate graph completeness while preserving aggregate reasons" - }, - { - "file": "gitnexus/src/cli/status.ts", + "file": "gitnexus/src/core/analysis-phase-supervisor.ts", "symbols": [], - "intended_change": "Expose graph freshness and vector/job readiness separately" - }, - { - "file": "gitnexus/src/cli/index.ts", - "symbols": [], - "intended_change": "Add total resource-budget flag; retain heap-only flag contract" + "intended_change": "NEW proposed filesystem/IPC phase supervisor; use actual child exit, existing lock acquisition and bounded result projection." }, { "file": "gitnexus/src/cli/analyze.ts", "symbols": [ - "resolveBudgetHeap", - "ensureHeap" + "ensureHeap", + "analyzeCommandImpl" ], - "intended_change": "Budget propagation and phase-specific progress" + "intended_change": "Compose supervision with heap respawn; keep live pipeline-result work in graph child and avoid extra forks for no-op/graph-only runs." + }, + { + "file": "gitnexus/src/cli/index.ts", + "symbols": [], + "intended_change": "Expose --resource-budget and pass immutable allocations without changing --memory-budget." + }, + { + "file": "gitnexus/src/cli/analyze-watch.ts", + "symbols": [ + "watchCommandWithRunnerIdentity" + ], + "intended_change": "Preserve heap flags and safe lifecycle across repeated analyses." + }, + { + "file": "gitnexus/src/server/analyze-worker-core.ts", + "symbols": [ + "runWorkerAnalysis" + ], + "intended_change": "Emit graph-prepared handoff separately from terminal completion; retain exit-only native cleanup." + }, + { + "file": "gitnexus/src/server/analyze-worker-ipc.ts", + "symbols": [ + "projectAnalyzeResultForIpc" + ], + "intended_change": "Allowlist bounded phase receipts/counters; never serialize pipelineResult." + }, + { + "file": "gitnexus/src/server/analyze-launch.ts", + "symbols": [ + "createLaunchAnalysisWorker" + ], + "intended_change": "Wait for graph-worker exit, finish publication, refresh graph backend, then continue embeddings; exactly one terminal event." + }, + { + "file": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", + "symbols": [], + "intended_change": "Preserve exit-based settlement and cooperative cancellation; carry nonterminal graph readiness." + }, + { + "file": "gitnexus/src/core/embeddings/embedding-job.ts", + "symbols": [ + "runPublishedEmbeddingJob" + ], + "intended_change": "Separate job identity from unpublished artifact ownership, reuse pending paid groups across rebases, and admit only after native release." + }, + { + "file": "gitnexus/src/core/embeddings/embedding-job-identity.ts", + "symbols": [ + "embeddingJobIdentity" + ], + "intended_change": "Keep graph-bound scheduling identity separate from reusable input/vector-space identities." + }, + { + "file": "gitnexus/src/core/embeddings/embedding-job-runner.ts", + "symbols": [ + "runEmbeddingJob" + ], + "intended_change": "Validate/reuse complete groups in the current manifest, preserve durability-before-acknowledgement and bounded queues." + }, + { + "file": "gitnexus/src/core/lbug/vector-store.ts", + "symbols": [ + "VectorStore" + ], + "intended_change": "Allow an exclusively owned unpublished artifact to outlive one graph-bound journal; retain complete group proofs and immutable published stores." + }, + { + "file": "gitnexus/src/storage/embedding-job-journal.ts", + "symbols": [ + "EmbeddingJobJournal" + ], + "intended_change": "Crash-safe rebase receipt and fresh manifest/cursor; no in-memory history chain or loss of acknowledged compatible groups." + }, + { + "file": "gitnexus/src/storage/embedding-job.ts", + "symbols": [], + "intended_change": "Owned filesystem artifacts and durable adoption/rebase records." + }, + { + "file": "gitnexus/src/core/embeddings/vector-publication.ts", + "symbols": [ + "publishVectorGeneration" + ], + "intended_change": "Verify current manifest coverage and close durability before publishing the artifact pointer." + }, + { + "file": "gitnexus/src/storage/repo-meta.ts", + "symbols": [ + "RepoMeta" + ], + "intended_change": "Version optional graph mode, job/artifact references, preserved coverage and recorded legacy provenance without inventing missing identities." + }, + { + "file": "gitnexus/src/storage/vector-generation.ts", + "symbols": [ + "selectVectorGeneration", + "metadataMatchesGraph" + ], + "intended_change": "Reject dirty/collapsed graphs; distinguish verified-current, preserved historical, legacy and pending states." + }, + { + "file": "gitnexus/src/storage/index-capabilities.ts", + "symbols": [], + "intended_change": "Expose availability separately from current-generation coverage; preserve existing result shapes with additive fields." + }, + { + "file": "gitnexus/src/storage/embedding-job-progress.ts", + "symbols": [], + "intended_change": "Represent preserved coverage and durable pending progress truthfully." + }, + { + "file": "gitnexus/src/core/embedding-checkpoint.ts", + "symbols": [ + "decideEmbeddingResume" + ], + "intended_change": "Retain legacy proof/force/drop policy; share one caller decision for new jobs." }, { "file": "gitnexus/src/cli/embeddings-sync.ts", "symbols": [ "embeddingsSyncCommand" ], - "intended_change": "Use shared durable embedding job" + "intended_change": "Route to the common bounded controller after ownership and graph validation." }, { - "file": "gitnexus/src/core/ingestion/workers/worker-pool.ts", + "file": "gitnexus/src/server/api.ts", "symbols": [ - "resolveWorkerHeapCapMb", - "createWorkerPool" + "createServer" ], - "intended_change": "Respect explicit phase allocation; preserve safe drain" + "intended_change": "Route /api/embed through the controller and carry safe readiness/progress fields." + }, + { + "file": "gitnexus/src/core/shared-store-analyze.ts", + "symbols": [ + "publishSharedGraph" + ], + "intended_change": "Publish graph facts independently of checkout-owned vector debt; preserve shared immutable graph ownership." + }, + { + "file": "gitnexus/src/storage/shared-store.ts", + "symbols": [ + "resolveGraphPath" + ], + "intended_change": "Resolve explicit graph mode and pin artifacts referenced by prepared publication, active job or readers." + }, + { + "file": "gitnexus/src/storage/repo-manager.ts", + "symbols": [ + "saveMeta" + ], + "intended_change": "Preserve owned generation references across metadata and registry updates." + }, + { + "file": "gitnexus/src/core/analysis-resource-budget.ts", + "symbols": [], + "intended_change": "Include supervisor and child process peaks; tune automatic allocations using matched-resource measurements." + }, + { + "file": "gitnexus/src/core/lbug/lbug-adapter.ts", + "symbols": [ + "drainAnalysisNativeResources" + ], + "intended_change": "Keep strict admission for genuinely in-process callers; never use it to override exit-only graph cleanup." + }, + { + "file": "gitnexus/src/core/embeddings/embedding-sidecar-client.ts", + "symbols": [ + "drainEmbeddingSidecarForAnalysis" + ], + "intended_change": "Wire prior-model drain before parser admission, while keeping no-op analysis lazy." + }, + { + "file": "gitnexus/src/core/analysis-progress.ts", + "symbols": [], + "intended_change": "Carry graph and embedding phase counters from actual durable producer events." + }, + { + "file": "gitnexus/src/core/embeddings/types.ts", + "symbols": [ + "EmbeddingProgress" + ], + "intended_change": "Maintain additive safe progress contracts." }, { "file": "gitnexus/src/mcp/local/local-backend.ts", "symbols": [ "LocalBackend.semanticSearch" ], - "intended_change": "Gate on complete matching vectors before model work; hydrate same-generation graph" + "intended_change": "Gate vector selection before model/native vector work; preserve BM25 parallelism and matching graph hydration." }, { - "file": "gitnexus/src/server/api.ts", + "file": "gitnexus/src/core/embeddings/vector-generation-search.ts", "symbols": [], - "intended_change": "Use same embedding job and generation-aware vector search" + "intended_change": "Serve a completed preserved artifact with explicit coverage state; never serve the active writer." }, { - "file": "gitnexus/src/server/analyze-job.ts", + "file": "gitnexus/src/core/index-freshness.ts", "symbols": [ - "AnalyzeJobProgress" + "getGraphIncompleteReasons" ], - "intended_change": "Optional counters without new terminal status values" + "intended_change": "Keep vector debt separate; dirty in-place graph mutation remains unavailable/unverified." + }, + { + "file": "gitnexus/src/cli/status.ts", + "symbols": [], + "intended_change": "Report graph freshness, embedding phase and preserved/current coverage independently." }, { "file": "gitnexus/src/server/ops-snapshot.ts", - "symbols": [ - "publicOpsProgress" - ], - "intended_change": "Explicit safe numeric/enumerated progress serialization" + "symbols": [], + "intended_change": "Allowlist public progress; omit paths, tokens and internal identities." }, { - "file": "gitnexus/src/core/analysis-publication.ts", + "file": "gitnexus/src/server/analyze-job.ts", "symbols": [], - "intended_change": "NEW: graph publication and crash reconciliation" + "intended_change": "Represent nonterminal graph readiness and one terminal outcome." }, { - "file": "gitnexus/src/core/analysis-resource-budget.ts", + "file": "gitnexus/src/server/repo-projection.ts", "symbols": [], - "intended_change": "NEW: pure resource allocation and admission policy" + "intended_change": "Expose additive coverage/readiness fields." }, { - "file": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", + "file": "gitnexus/src/mcp/resources.ts", "symbols": [], - "intended_change": "NEW: durable deterministic work manifest and bounded source" - }, - { - "file": "gitnexus/src/core/embeddings/embedding-job.ts", - "symbols": [], - "intended_change": "NEW: shared resumable execution protocol" - }, - { - "file": "gitnexus/src/core/embeddings/vector-store.ts", - "symbols": [], - "intended_change": "NEW: isolated vector-only native adapter" - }, - { - "file": "gitnexus/src/storage/embedding-job.ts", - "symbols": [], - "intended_change": "NEW: filesystem-only compact job journal" + "intended_change": "Expose the same graph/vector state in repository context." }, { "file": "gitnexus/bench/analyze-resume/measure.mjs", "symbols": [], - "intended_change": "NEW: paired throughput/memory/restart/query harness" + "intended_change": "NEW proposed end-to-end paired harness; existing microbenchmarks do not replace it." + }, + { + "file": "gitnexus/bench/analyze-resume/README.md", + "symbols": [], + "intended_change": "Document raw samples, environment, pair ordering, equal-work checks and limits." }, { "file": "RUNBOOK.md", "symbols": [], - "intended_change": "Document phase recovery, new resource option and limits" + "intended_change": "Document phase recovery, resource flag, preserved coverage, migration and downgrade behavior." } ], "tests": [ { - "file": "gitnexus/test/unit/embedding-pipeline.test.ts", + "file": "gitnexus/test/unit/analysis-publication.test.ts", + "status": "existing", "scenarios": [ - "Existing: interrupted-window hashes, before-write intent, fallback and rejection isolation, bounded expansion" - ] - }, - { - "file": "gitnexus/test/unit/embedding-checkpoint.test.ts", - "scenarios": [ - "Existing: identity/force/drop/legacy cases; visible failed retry exhaustion" - ] - }, - { - "file": "gitnexus/test/unit/analyze-memory-budget.test.ts", - "scenarios": [ - "Existing: heap-only CLI flag compatibility" - ] - }, - { - "file": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", - "scenarios": [ - "Existing: completed vector reuse, second crash, changed/deleted nodes, model mismatch and drop" - ] - }, - { - "file": "gitnexus/test/integration/embedding-identity-integrity.test.ts", - "scenarios": [ - "Existing: exact identities and complete unique chunk sets" - ] - }, - { - "file": "gitnexus/test/unit/local-backend-lazy-vector.test.ts", - "scenarios": [ - "Existing: pending/mismatch causes no model or vector requests" - ] - }, - { - "file": "gitnexus/test/unit/server-sse-payload.test.ts", - "scenarios": [ - "Existing: additive safe counters and exactly one honest terminal event" - ] - }, - { - "file": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", - "scenarios": [ - "Existing: safe sibling reuse/publication and pending-job pinning" - ] - }, - { - "file": "gitnexus/test/unit/embedding-work-manifest.test.ts", - "scenarios": [ - "NEW: lazy large corpus, duplicate IDs, failed labels, stable byte-bounded units" - ] - }, - { - "file": "gitnexus/test/unit/analysis-resource-budget.test.ts", - "scenarios": [ - "NEW: cgroups, overrides, native floors, phase allocations and watch propagation" + "In-place mutation invalidates generation before writes; no full graph copy/hash on a tiny edit.", + "Staged crash keeps old graph; in-place crash preserves existing dirty-recovery behavior; after publication embedding failures preserve new graph." ] }, { "file": "gitnexus/test/integration/analyze-phase-publication.test.ts", + "status": "new", "scenarios": [ - "NEW: before/after graph publication crashes and cold/warm reader availability" + "Graph child sends prepared then stays alive: zero provider/open-new-pool calls until exit.", + "Cold/warm MCP and HTTP readers see new graph/FTS while separate vector writer is open.", + "Crash before/after file and metadata publication; competing writer changes slot/generation during handoff: reject stale receipt without paid requests.", + "Registry-name collision/change and --skills/self-commit: context names match final registration; prepared slot never appears graph-ready." + ] + }, + { + "file": "gitnexus/test/unit/analyze-worker-core.test.ts", + "status": "existing", + "scenarios": [ + "Graph-prepared is nonterminal; cancellation owns terminal slot; preserve skipNativeCloseOnExit." + ] + }, + { + "file": "gitnexus/test/unit/analyze-worker-ipc.test.ts", + "status": "existing", + "scenarios": [ + "Graph getters throw if traversed: bounded phase receipt still serializes; safe counters only." + ] + }, + { + "file": "gitnexus/test/unit/analyze-launch-collapse.test.ts", + "status": "existing", + "scenarios": [ + "Collapse rejected before backend publication; no embedding child or model admission." + ] + }, + { + "file": "gitnexus/test/unit/analyze-launch-branch-settle.test.ts", + "status": "existing", + "scenarios": [ + "External/shared/branch slot from result is revalidated after child exit." + ] + }, + { + "file": "gitnexus/test/unit/auto-sync-analysis-worker.test.ts", + "status": "existing", + "scenarios": [ + "Prepared IPC is not terminal; actual exit gates completion; cancelled native child is not force-killed." + ] + }, + { + "file": "gitnexus/test/integration/embedding-job-controller.test.ts", + "status": "existing", + "scenarios": [ + "Unchanged graph skips parsing and complete provider requests; graph changes rebase pending artifact.", + "Two source changes between interruptions retain first-attempt paid groups and previous completed fallback.", + "New model/text/dimensions reject incompatible reuse before requests; smaller resource budget does not invalidate units." + ] + }, + { + "file": "gitnexus/test/integration/embedding-vector-store.test.ts", + "status": "existing", + "scenarios": [ + "Current manifest validates complete groups; stale/deleted nodes pruned before publish; published store never mutated.", + "Rebase crash at intent/adoption/checkpoint boundaries preserves owned paid artifact and rejects foreign identity." + ] + }, + { + "file": "gitnexus/test/unit/embedding-job-journal.test.ts", + "status": "existing", + "scenarios": [ + "One active unit; corrupted checksum/missing segment preserved and recovered only from complete native groups." ] }, { "file": "gitnexus/test/integration/embedding-job-restart.test.ts", + "status": "new", "scenarios": [ - "NEW: interruption at intent/write/durable/cursor/HNSW/pointer boundaries; at most active unit replay" + "Real child crashes at JS-visible intent/native-write/checkpoint/cursor/HNSW/pointer boundaries; at most active unit replay.", + "ENOSPC, cancellation and repeated interruption keep published graph available and no duplicate/missing chunk rows." ] }, { "file": "gitnexus/test/integration/embedding-resource-bounds.test.ts", + "status": "new", "scenarios": [ - "NEW: lazy 10k/100k/1m descriptors and real-native bounded-retention check" + "Lazy 10k/100k/1m descriptors and oversized Unicode nodes: fixed queue high-water mark, identical text/chunks, no corpus materialization.", + "Smaller real-native case counts parent, graph/embedding child and sidecar peaks without double-counting thread RSS." + ] + }, + { + "file": "gitnexus/test/unit/vector-generation.test.ts", + "status": "existing", + "scenarios": [ + "Dirty/collapsed/mismatched graphs gate before native/model access; historical completed artifact is labeled preserved; pending job disables vector lane." + ] + }, + { + "file": "gitnexus/test/unit/local-backend-vector-generation.test.ts", + "status": "existing", + "scenarios": [ + "Preserved results hydrate current IDs and reject deleted nodes; active vector writer never opened; query ranking parity." + ] + }, + { + "file": "gitnexus/test/unit/embedding-job-progress.test.ts", + "status": "existing", + "scenarios": [ + "Completed+reused+pending+failed=total; active subset of pending; preserved is not current-complete; no false terminal event." + ] + }, + { + "file": "gitnexus/test/unit/server-sse-payload.test.ts", + "status": "existing", + "scenarios": [ + "Forward producer/IPC phase details through safe allowlist; graph-ready nonterminal; exactly one final success/failure/cancel event." + ] + }, + { + "file": "gitnexus/test/unit/analyze-memory-budget.test.ts", + "status": "existing", + "scenarios": [ + "Heap-only flag retains old meaning, including explicit ambient heap and respawn opt-out." + ] + }, + { + "file": "gitnexus/test/unit/analysis-resource-budget.test.ts", + "status": "existing", + "scenarios": [ + "Supervisor+children+native pools+sidecar+queues fit; defaults preserve useful concurrency when feasible." + ] + }, + { + "file": "gitnexus/test/unit/lbug-analysis-resources.test.ts", + "status": "existing", + "scenarios": [ + "Failed close remains charged; session drain blocks model/parser; cancellation during drain makes no metadata/provider calls." + ] + }, + { + "file": "gitnexus/test/integration/embedding-native-release.test.ts", + "status": "existing", + "scenarios": [ + "Explicit native pools are not reused before verified physical close; failed close retries, result-close failures clear active admission." + ] + }, + { + "file": "gitnexus/test/unit/embedding-sidecar-client.test.ts", + "status": "existing", + "scenarios": [ + "Parser phase waits for prior model-child exit; cancellation does not release a physically active request early." + ] + }, + { + "file": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", + "status": "existing", + "scenarios": [ + "Keep legacy complete-group reuse, second-crash, changed/deleted nodes, force/drop behavior and known-identity relocation without provider requests." ] }, { "file": "gitnexus/test/integration/vector-generation-compatibility.test.ts", + "status": "new", "scenarios": [ - "NEW: legacy/shared/external/branch storage; Windows; concurrent writers; missing extensions" + "Plain analyze preserves legacy and previously completed vector coverage without automatic requests.", + "Unknown legacy provenance remains unknown; explicit generation logs any unsafe-reuse invalidation; relocation alone never invalidates verified groups.", + "Windows open handles, no VECTOR/FTS, shared sibling and external/branch storage, concurrent writers." + ] + }, + { + "file": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", + "status": "existing", + "scenarios": [ + "Prepared publication/pending vector work pins its artifacts; sibling cannot clean or mutate them." ] } ], "verification_commands": [ "cd gitnexus && npm run build", "cd gitnexus && npx tsc --noEmit", - "cd gitnexus && npm test -- test/unit/embedding-pipeline.test.ts test/unit/embedding-checkpoint.test.ts test/unit/analyze-memory-budget.test.ts test/unit/local-backend-lazy-vector.test.ts test/unit/server-sse-payload.test.ts", - "cd gitnexus && npm run test:integration -- test/integration/analyze-staged-embedding-recovery.test.ts test/integration/embedding-identity-integrity.test.ts test/integration/shared-store-embedding-reuse.test.ts", + "cd gitnexus && npm run typecheck:tests", + "cd gitnexus && npm test -- test/unit/analysis-publication.test.ts test/unit/vector-generation.test.ts test/unit/analyze-worker-core.test.ts test/unit/analyze-worker-ipc.test.ts test/unit/embedding-job-journal.test.ts test/unit/analysis-resource-budget.test.ts test/unit/lbug-analysis-resources.test.ts test/unit/server-sse-payload.test.ts", + "cd gitnexus && npm run test:integration -- test/integration/embedding-job-controller.test.ts test/integration/embedding-vector-store.test.ts test/integration/embedding-native-release.test.ts test/integration/analyze-staged-embedding-recovery.test.ts test/integration/shared-store-embedding-reuse.test.ts", "cd gitnexus && npm test", "cd gitnexus && node --import tsx bench/embedding-inputs/measure.cjs --check", "cd gitnexus && node --import tsx bench/embedding-identities/measure.cjs --check" ], "risks": [ - "CRITICAL coordinator/query and HIGH resume impact require gate acknowledgement; graph expansion has UID inconsistencies.", - "Native reader/writer lifetime and publication crash windows; Windows handles.", - "Additional vector pool and manifest I/O may regress performance; measure before integration.", - "Large-node chunking and native query materialization may exceed budgets.", - "Legacy/shared-store compatibility; incomplete vectors must never appear complete." + "CRITICAL decideEmbeddingResume across analyze/sync/API; HIGH semanticSearch; previously CRITICAL native close/session helpers. Re-run impact before each edit.", + "Process handoff, publication, historical-coverage semantics, migration and rebase durability must activate coherently.", + "New process startup, manifest/native reopen, HTTP response parsing or pool sizing may regress latency/throughput; performance gate is unresolved." ], "assumptions": [ - "Prove separate vector database feasibility and throughput with real-native concurrent-reader tests at the same budget before integration.", - "Inspect native query plan and scaling to prove bounded manifest reads avoid repeated label scans and unbounded native content retention.", - "Verify large-node input equivalence and peak memory; route back if chunker redesign is required.", - "Verify Windows and local-model execution environments; disclose unavailable validation." + "Whole-command performance is unmeasured. Verify with at least seven alternating pairs at equal work/resources before activation/shipment; no allowed slowdown threshold.", + "Automatic resource defaults are provisional. Include supervisor RSS, native pools, child/model runtime, queues and headroom; tune using the full matrix.", + "Windows and a representative local model have not been validated in this environment. Obtain those executions or report the acceptance gap; never infer a pass from Linux mocks.", + "PDG exists but current close slice is unavailable and controller output has mismatched source offsets plus truncation. Treat statement-level coverage as unresolved; source constraints remain authoritative.", + "Legacy model/text provenance may be absent. Preserve legacy read/preserve-only behavior; never invent a verified identity. Explicit new generation may invalidate only groups whose required compatibility cannot be proven, with reason and cost visible." ], "open_questions": [ - "Determine budget/queue/concurrency defaults from measurements; no performance claim is established.", - "Refresh graph with source-current analyzer in execution and confirm/fallback on symbol UID mismatches.", - "Validate downgrade behavior for new generation pointers before enabling the format." + "No user product preference is required beyond the revised architecture gate. Performance defaults are decided by measurements, not an arbitrary regression allowance.", + "Windows/local-model evidence and PDG coverage limits remain explicit acceptance gaps." ], "avoid": [ - "Do not repeat full repository discovery; use this context and targeted drift checks.", - "Do not modify pre-existing CI/eval/benchmark changes or untracked tooling files.", - "Do not edit code before the LFG gate and fresh impact analysis; do not commit before complete detect_changes and required tests.", - "Do not interpret UNKNOWN or the inconsistent current graph as proof of low impact.", - "Do not replace memory-budget heap semantics or bypass native COPY safety floors.", - "Do not materialize corpus content/hash maps; do not use repeated OFFSET pagination.", - "Do not claim streaming JavaScript alone bounds native query allocations.", - "Do not terminate native-active workers or use the singleton streamQuery with its WAL driver active.", - "Do not publish partial vectors as complete, silently truncate content, or omit failed work.", - "Do not re-embed completed compatible units, copy the full graph for each embedding job, or rebuild HNSW per unit.", - "Do not discard legacy vectors before successful migration/publication.", - "Do not weaken performance acceptance, remove features, change baselines to mask regressions, or claim unmeasured results.", - "Do not alter CI/release/security config, push, or publish a PR as part of this plan." + "Do not redo landed helper commits or repeat full repository discovery.", + "Do not force physical graph immutability/copying for default private incremental updates.", + "Do not invoke real native graph close on a CLI/server exit-only path just to begin embeddings; wait for process exit.", + "Do not serialize KnowledgeGraph/pipelineResult through IPC or add unbounded job-history scans.", + "Do not create a borrowed-index-lock token protocol; reacquire and revalidate the existing exclusive lock at handoff.", + "Do not mutate published vector artifacts, expose an active writer to readers, or present preserved historical coverage as current complete.", + "Do not charge provider work on plain preserve-only analyze, fabricate legacy identities, truncate input, or accept isolated chunk hashes as complete groups.", + "Do not change --memory-budget semantics or invalidate paid work because of scheduling/resource changes.", + "Do not terminate a native-active thread/worker; use cooperative safe points and verified exit.", + "Do not use microbenchmarks as proof of end-to-end non-regression or compare unequal work/resources.", + "Do not edit unrelated /workspace changes, dependencies, language-resolution behavior, CI/release/security config; no push or PR." + ], + "completed_commits": [ + { + "commit": "f0f3b9abe", + "result": "Original approved plan; superseded only by this Deepen revision." + }, + { + "commit": "60ddccefa", + "result": "Native storage feasibility and baseline microbenchmarks; whole-command harness still missing." + }, + { + "commit": "dbf62eab1", + "result": "Pure resource resolver and deterministic bounded manifest." + }, + { + "commit": "ec9063f65", + "result": "Vector store and durable segmented journal." + }, + { + "commit": "ba23148ba", + "result": "Bounded runner/text streaming and complete-group reuse." + }, + { + "commit": "084c52895", + "result": "Staged publication and generation-aware readers; in-place mode still needed." + }, + { + "commit": "f2fc492cf", + "result": "Published-job controller; no production callers yet." + }, + { + "commit": "8bdd572fd", + "result": "Optional parser/native resource allocation plumbing." + }, + { + "commit": "8dec4f273", + "result": "Physical sidecar reservations; parser admission not wired." + }, + { + "commit": "700d6aca8", + "result": "Readiness/progress consumers; producer/IPC wiring incomplete." + }, + { + "commit": "2363a1060", + "result": "Interrupted native bootstrap and journal recovery." + }, + { + "commit": "f953c569e", + "result": "Verified explicit native pool release and retry accounting." + }, + { + "commit": "8c6b680f3", + "result": "Main graph/session drain before in-process phase admission." + } + ], + "remaining_steps": [ + "Create full paired harness and re-anchor baseline", + "Extend graph publication and bounded exit handoff", + "Rebase pending artifacts and preserve legacy/historical coverage", + "Wire resource/progress/shared-store surfaces", + "Activate all three production writers together", + "Full regression/performance matrix and docs", + "GitNexus review; no push/PR" ], "index_refresh": { - "status": "skipped", - "reason": "Current index HEAD but analyzer-source equivalence unverified; graph UID mismatch; no PDG; another process owns local graph. Planning must not rebuild dist.", - "runner": "node .gitnexus/run.cjs -> npx gitnexus@latest", - "recorded_cli_version": "1.6.12", - "recorded_runtime": "v24.19.0", - "planning_runtime": "v22.23.3" + "runner": "node gitnexus/dist/cli/index.js", + "storage": "isolated local navigation index", + "head": "8c6b680f3071025189804ffebac2fd3bc8bf1232", + "schema": 4, + "status": "up-to-date", + "runner_identity_status": "current", + "content_drift": "current", + "incomplete_reasons": [], + "receipt_match": "Exact MCP context runnerIdentity equals local status.index.runnerIdentity; local build/source current.", + "deepen_refreshes": 0, + "limitations": [ + "Connected MCP native engine uses storage42, local index storage47; use build-current local backend for graph calls.", + "Clusters/processes resource attempts failed engine mismatch; bounded local impact process summaries used instead.", + "PDG source offsets/truncation remain unresolved; no unrelated indexer repair included." + ] }, - "validation_status": "Planning source review only; no production changes, tests, benchmarks or commits performed." + "validation_status": { + "feature": "incomplete; bounded controller not yet activated in production", + "latest_focused": "80 passed across six relevant suites for 8c6b680; build, source/test typecheck and lint passed (pre-existing warnings).", + "prior_focused": "f953c569: 265 passed, 1 skipped across recorded focused sets; 2363a106 bootstrap/journal regressions passed.", + "candidate_full_suite": "not run", + "performance": "microbenchmarks only; no whole-command non-regression acceptance", + "review": "gitnexus-review not run", + "platforms": "Linux native exercised; Windows/local-model unverified" + }, + "reanchor": { + "old_head": "f641342d1fcbfdcb764c5e478db0f69e6cea899a", + "new_head": "8c6b680f3071025189804ffebac2fd3bc8bf1232", + "old_plan_digest": "sha256:c36d6ad14288ca3ff0510c7d18a769d750c72475b5fff0fb476b10709e22d0e9", + "changed_cited_paths": [ + "gitnexus/src/cli/status.ts", + "gitnexus/src/core/analysis-publication.ts", + "gitnexus/src/core/analysis-resource-budget.ts", + "gitnexus/src/core/embeddings/embedding-job.ts", + "gitnexus/src/core/embeddings/embedding-pipeline.ts", + "gitnexus/src/core/embeddings/embedding-work-manifest.ts", + "gitnexus/src/core/embeddings/types.ts", + "gitnexus/src/core/index-freshness.ts", + "gitnexus/src/core/ingestion/workers/worker-pool.ts", + "gitnexus/src/core/lbug/lbug-adapter.ts", + "gitnexus/src/core/lbug/lbug-config.ts", + "gitnexus/src/core/lbug/pool-adapter.ts", + "gitnexus/src/core/run-analyze.ts", + "gitnexus/src/mcp/local/local-backend.ts", + "gitnexus/src/server/analyze-job.ts", + "gitnexus/src/server/api.ts", + "gitnexus/src/server/ops-snapshot.ts", + "gitnexus/src/storage/embedding-job.ts", + "gitnexus/src/storage/repo-manager.ts", + "gitnexus/src/storage/repo-meta.ts", + "gitnexus/src/storage/shared-store.ts", + "gitnexus/test/unit/analysis-resource-budget.test.ts", + "gitnexus/test/unit/embedding-work-manifest.test.ts", + "gitnexus/test/unit/server-sse-payload.test.ts" + ], + "policy": "Changed old ranges replaced by current seam evidence or retired; landed helpers reconciled rather than blindly re-pinned." + } } } ``` ## 12. Assumptions and Open Questions -- [assumed] A vector-only LadybugDB plus a published read-only graph can meet current latency and throughput. Prove with real-native cold/warm concurrent-reader benchmarks in step 1; the additional pool must fit the same resource envelope. -- [assumed] A single key-only scan plus bounded content fetches can avoid repeated full scans and unbounded native materialization. Inspect the installed engine's query behavior and measure scaling. Do not assume `ORDER BY id LIMIT` uses an index or that JavaScript streaming bounds native allocations. -- [assumed] Very large single-node chunking can preserve embedding input semantics within the configured envelope. Test actual chunker behavior before claiming compliance; if it requires a broader chunker redesign, deepen this plan rather than silently truncating or widening the budget. -- [assumed] Linux and Windows runners, plus a representative local embedding runtime, are available during execution. If unavailable, record the platform/runtime gap and do not declare its acceptance tests passed. -- **Open implementation gate:** exact budget defaults, queue byte caps and overlap/concurrency settings must come from measurements; preserve existing useful concurrency whenever memory permits. No performance improvement or non-regression has been measured during planning. -- **Evidence limitation:** matching graph HEAD does not repair the npm analyzer identity mismatch or inconsistent graph UIDs. Source is authoritative. Build-current graph/PDG verification must precede implementation; if the mismatch persists, record source-derived impact plus unresolved graph coverage. -- **Compatibility decision:** introduce `--resource-budget` for the whole analysis process family while preserving `--memory-budget`. Public metadata additions remain optional; legacy status aggregation remains available. Downgrade binaries that do not understand new generation pointers must fail clearly or use the retained legacy graph, never manufacture current freshness. -- **Working tree:** `.github/scripts/evolution-runner.py`, `eval/tests/test_evolution_runner.py`, and `gitnexus/bench/analyze-phase-breakdown.md` were already dirty; untracked tooling files also existed. They are outside this task. The evidence snapshot records them without including their contents in this plan. -- **Deferred:** distributed embedding workers, new database engines, provider-specific acceleration, full graph-ingestion out-of-core redesign, partial-vector serving, automatic background scheduling, and unrelated graph-identity repairs. They are not required to implement this phase boundary; if resource validation proves one necessary, re-plan explicitly. +- [assumed] Whole-command performance is unmeasured. Verify with at least seven alternating pairs at equal work/resources before activation/shipment; no allowed slowdown threshold. +- [assumed] Automatic resource defaults are provisional. Include supervisor RSS, native pools, child/model runtime, queues and headroom; tune using the full matrix. +- [assumed] Windows and a representative local model have not been validated in this environment. Obtain those executions or report the acceptance gap; never infer a pass from Linux mocks. +- [assumed] PDG exists but current close slice is unavailable and controller output has mismatched source offsets plus truncation. Treat statement-level coverage as unresolved; source constraints remain authoritative. +- [assumed] Legacy model/text provenance may be absent. Preserve legacy read/preserve-only behavior; never invent a verified identity. Explicit new generation may invalidate only groups whose required compatibility cannot be proven, with reason and cost visible. + +No new product preference is needed beyond approval of the concrete architecture revision. Implementation choices above are fixed; allocation tuning follows evidence. The full matrix, Windows/local-model coverage and final review remain completion gates, not reasons to claim a partial feature finished. + +The original stale-analyzer/graph-UID limitation has been replaced by a verified current local runner/index and explicit narrower PDG/resource limitations. Changed cited paths were reconciled before moving the evidence pin. Existing shared language-neutral ingestion rules and no-native-termination guardrails remain binding. + +Deferred: distributed embedding workers, new database engines, unrelated PDG fixes, graph-ingestion out-of-core redesign, automatic background scheduling and serving vectors from incomplete active jobs. Preserving a previously completed artifact with historical-coverage labeling is explicitly in scope for backward compatibility. + +The original `/workspace` user edits are outside this worktree/task. Do not stage them. No automatic push/PR. ## 13. Definition of Done -- [ ] Graph/keyword publication is independently durable and visible to new readers before embedding completes; failures preserve that new graph. -- [ ] At fixed configuration, retained input/chunk/vector queues remain bounded as corpus size grows, including large nodes and failed work; total-resource reservations and observed memory are reported honestly. -- [ ] Compatible retries skip parse/graph rebuilding and completed embedding requests; only unfinished work plus at most the active interrupted unit is replayed. -- [ ] Every crash boundary passes with exact complete chunk sets, unique keys and no lost acknowledged work; invalidation reasons explain retained versus recomputed work. -- [ ] Graph freshness, embedding coverage and vector readiness are distinct across CLI, MCP, HTTP and SSE; pending/failed vectors cannot masquerade as complete. -- [ ] Legacy indexes/checkpoints, explicit force/drop, shared stores, external/branch storage and Windows readers retain safe behavior. -- [ ] Paired performance evidence shows no repeatable regression at comparable resources, with fingerprints proving equivalent work. Unmeasured platforms and inconclusive results remain disclosed limitations. -- [ ] Required tests/typecheck pass, graph impact/change analysis is complete, and the LFG review verdict and remaining findings are reported. No unrelated edits, push or PR publication. +- [ ] New graph/keyword results are independently durable and readable before embeddings finish; every embedding failure preserves that graph. +- [ ] Staged and in-place crash semantics are tested and documented; tiny private incremental updates do not gain a whole-graph copy/hash. +- [ ] Exit-only graph workers physically exit before embedding admission; no native-active termination, unbounded IPC or double-counted resource reservations. +- [ ] All three production writers use bounded input/chunk/vector queues with identical text/coverage semantics; large-corpus and oversized-node cases pass. +- [ ] Compatible unchanged retries skip graph rebuilding and acknowledged provider requests; repeated source-change interruptions preserve all provably reusable paid groups. +- [ ] Plain analyze preserves existing vectors with zero automatic requests; current/preserved/legacy/pending coverage is truthful across CLI, MCP, HTTP and SSE. +- [ ] Force/drop, legacy recovery, shared/external/branch storage and Windows handle behavior pass their required cases; unmeasured platforms remain explicitly unresolved. +- [ ] Paired full-command evidence demonstrates no repeatable throughput/query-latency regression at comparable resources, including all added work and migration costs. +- [ ] Candidate full checks and final GitNexus review are complete; remaining findings are reported. No unrelated edits or unrequested publication. From a463cc3940459d67d0eef7c8ae46b4c2dab0b086 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 3 Nov 2026 18:34:43 +0000 Subject: [PATCH 15/67] perf(bench): measure paired analyze runs --- gitnexus/bench/analyze-resume/README.md | 45 ++ gitnexus/bench/analyze-resume/measure.mjs | 652 ++++++++++++++++++ .../bench/analyze-resume/measure.test.mjs | 127 ++++ 3 files changed, 824 insertions(+) create mode 100644 gitnexus/bench/analyze-resume/measure.mjs create mode 100644 gitnexus/bench/analyze-resume/measure.test.mjs diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 18838031d..8618ce6d1 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -24,6 +24,51 @@ end-to-end analyze throughput, crash recovery, model throughput, or total RSS bounds. Run the application regression matrix before making those claims. Windows handle behavior and local-model execution also need their own runs. +Measure complete CLI invocations with two built checkouts: + +```sh +node bench/analyze-resume/measure.mjs \ + --baseline /path/to/baseline/gitnexus/dist/cli/index.js \ + --output /tmp/analyze-pairs.json +``` + +The harness creates isolated Git fixtures, registries and stores, alternates +baseline/candidate order, warms each scenario and records seven measured pairs. +Stage both built runtimes on the same filesystem with matching dependencies. +Preflight rejects different runtime devices/filesystem types, dependency +lockfiles or native entry bytes; each completed pair also compares the full +resolved dependency digest from its analyzer receipts. A checkout on a remote +filesystem compared with a local checkout is not useful regression evidence. +It includes CLI startup, graph writes, embeddings and final index work. Scenarios +cover graph-only, warm no-op, one-function incremental edits, fresh/cached HTTP +embeddings and resume after a deterministic provider outage. The outage ends +normally through application error handling; it does not kill native work. +`--help` lists corpus, heap, worker and scenario options. Both sides receive the +same explicit allocations; default-budget behavior needs a separate comparison. + +Raw samples include command/log paths, analyzer receipts, graph/vector counts, +provider request/input counts, deterministic input fingerprints and sampled Linux +process-family RSS. Fixture setup is outside the timer and recorded separately. +Logs and measurements remain beside the output; native stores are removed after +each measurement unless `--keep-stores` is set. An existing output is never +overwritten. No-op/cached runs and equal-work checks fail on unexpected work. +Sending SIGINT to the harness PID stops between invocations after the active CLI +exits naturally; the harness does not forward a termination signal to native work. + +Results use paired log-ratio bootstrap intervals and report regression, +inconclusive, or no detected regression. Fewer than seven pairs is always +smoke-only. The report always marks overall acceptance incomplete: synthetic +counts/input parity do not establish query ranking, local-model performance, +concurrent query latency, exact peak RSS, queue/native reservations, CPU/disk costs, +default-budget behavior or Windows correctness. Those remain part of the final +regression/performance matrix. Do not run measurements alongside indexing/tests. + +Check the harness itself with: + +```sh +node --test bench/analyze-resume/measure.test.mjs +``` + Measure the implemented vector store and journal after building: ```sh diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs new file mode 100644 index 000000000..d96745c77 --- /dev/null +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -0,0 +1,652 @@ +/** Whole-command paired measurements. Run --help; never treats a smoke run as acceptance. */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn, execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { createRequire } from 'node:module'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +const SCENARIOS = ['graph-only', 'noop', 'incremental', 'http-fresh', 'http-cached', 'http-resume']; +const sha = (text) => createHash('sha256').update(text).digest('hex'); + +async function runtimeEnvironment(cli) { + const describeFile = async (file) => { + const resolved = await fs.realpath(file); + const [stat, filesystem] = await Promise.all([ + fs.stat(resolved, { bigint: true }), + fs.statfs(resolved, { bigint: true }), + ]); + return { path: resolved, device: String(stat.dev), filesystem: String(filesystem.type) }; + }; + const nativeEntry = createRequire(cli).resolve('@ladybugdb/core'); + return { + cli: await describeFile(cli), + nativeEntry: await describeFile(nativeEntry), + nativeEntryDigest: sha(await fs.readFile(nativeEntry)), + lockDigest: sha(await fs.readFile(path.resolve(path.dirname(cli), '../../package-lock.json'))), + }; +} + +export function assertComparableRuntimes(baseline, candidate) { + for (const component of ['cli', 'nativeEntry']) { + assert.equal( + candidate[component].device, + baseline[component].device, + `${component} runtime files are on different devices; stage both checkouts on the same filesystem`, + ); + assert.equal( + candidate[component].filesystem, + baseline[component].filesystem, + `${component} runtime files are on different filesystem types`, + ); + } + assert.equal(candidate.lockDigest, baseline.lockDigest, 'Dependency lockfiles differ'); + assert.equal( + candidate.nativeEntryDigest, + baseline.nativeEntryDigest, + 'Native runtime entry bytes differ', + ); +} + +export function assertCompletedCase(scenario, metadata, beforeMetadata) { + assert( + !metadata.incrementalInProgress && !metadata.graphWriteCollapsed, + 'Graph was not finalized', + ); + assert( + Number.isSafeInteger(metadata.stats?.nodes) && metadata.stats.nodes > 0, + 'Missing graph node count', + ); + if (scenario === 'noop') { + assert(beforeMetadata?.indexedAt, 'No-op setup lacks a freshness receipt'); + assert.equal(metadata.indexedAt, beforeMetadata.indexedAt, 'No-op changed graph freshness'); + } + if (scenario.startsWith('http-')) { + assert(metadata.stats.embeddings > 0, 'Embedding scenario produced no vectors'); + assert( + !metadata.embeddingCheckpoint && !metadata.embeddingJob, + 'Embedding work remains pending', + ); + } +} +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +/** Paired log-ratio bootstrap. An interval crossing 1 is unresolved, never a pass. */ +export function summarizePairs(pairs) { + assert(pairs.length > 0, 'At least one measured pair is required'); + const logs = pairs.map(({ baseline, candidate }) => { + assert( + Number.isFinite(baseline) && baseline > 0 && Number.isFinite(candidate) && candidate > 0, + ); + return Math.log(candidate / baseline); + }); + const average = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length; + let seed = 3598; + const random = () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return seed / 2 ** 32; + }; + const samples = Array.from({ length: 10_000 }, () => + Math.exp(average(logs.map(() => logs[Math.floor(random() * logs.length)]))), + ).sort((a, b) => a - b); + const interval = [samples[250], samples[9749]]; + return { + pairs: pairs.length, + baselineMedianMs: median(pairs.map((p) => p.baseline)), + candidateMedianMs: median(pairs.map((p) => p.candidate)), + geometricRatio: Math.exp(average(logs)), + pairedBootstrap95: interval, + evidence: + pairs.length < 7 + ? 'smoke-only' + : interval[0] > 1 + ? 'regression' + : interval[1] < 1 + ? 'no-detected-regression' + : 'inconclusive', + }; +} + +/** Sample Linux process RSS once per process, including descendants, never per thread. */ +async function sampleTree(pid) { + if (process.platform !== 'linux') return undefined; + const pending = [pid]; + const seen = new Set(); + let bytes = 0; + while (pending.length) { + const current = pending.pop(); + if (seen.has(current)) continue; + seen.add(current); + try { + const [status, children] = await Promise.all([ + fs.readFile(`/proc/${current}/status`, 'utf8'), + fs.readFile(`/proc/${current}/task/${current}/children`, 'utf8'), + ]); + bytes += Number(status.match(/^VmRSS:\s+(\d+)\s+kB$/m)?.[1] ?? 0) * 1024; + pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number)); + } catch (error) { + if (!['ENOENT', 'ESRCH'].includes(error.code)) throw error; + } + } + return bytes; +} + +export async function measureProcess(command, args, { cwd, env, logFile, sampleMs = 25 }) { + const log = await fs.open(logFile, 'wx'); + let timer; + let polling; + let samplingError; + let peakRssBytes = process.platform === 'linux' ? 0 : null; + let samples = 0; + const start = performance.now(); + try { + const child = spawn(command, args, { cwd, env, stdio: ['ignore', log.fd, log.fd] }); + const poll = () => { + if (!polling && child.pid) { + polling = sampleTree(child.pid) + .then((rss) => { + if (rss !== undefined) { + peakRssBytes = Math.max(peakRssBytes, rss); + samples++; + } + }) + .catch((error) => { + samplingError ??= error.message; + }) + .finally(() => { + polling = undefined; + }); + } + }; + poll(); + timer = setInterval(poll, sampleMs); + const outcome = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (exitCode, signal) => resolve({ exitCode, signal })); + }); + const elapsedMs = performance.now() - start; + clearInterval(timer); + await polling; + return { + ...outcome, + elapsedMs, + sampledPeakProcessTreeRssBytes: samplingError ? null : peakRssBytes, + rssSamples: samples, + rssSampleIntervalMs: sampleMs, + samplingError, + logFile, + }; + } finally { + clearInterval(timer); + await polling; + await log.close(); + } +} + +export async function startProvider(dimensions = 256) { + let active; + const server = createServer(async (req, res) => { + const counters = active; + try { + assert(counters && req.method === 'POST', 'Unexpected provider request'); + let size = 0; + const chunks = []; + for await (const chunk of req) { + size += chunk.length; + assert(size <= 8 * 1024 * 1024, 'Benchmark request exceeds 8 MiB'); + chunks.push(chunk); + } + const { input } = JSON.parse(Buffer.concat(chunks).toString('utf8')); + assert( + Array.isArray(input) && input.length && input.every((text) => typeof text === 'string'), + ); + counters.requests++; + counters.requestBytes += size; + if (counters.acceptedInputs + input.length > counters.failAfter) { + counters.rejectedRequests++; + res.writeHead(503, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'Deterministic interrupted-run fixture' } })); + return; + } + const data = input.map((text, index) => { + const hash = createHash('sha256').update(text).digest(); + counters.inputs.add(hash.toString('hex')); + counters.acceptedInputs++; + // Same text produces exactly the same nonzero vector in either implementation. + const embedding = Array.from({ length: dimensions }, (_, i) => (hash[i % 32] + 1) / 256); + return { object: 'embedding', index, embedding }; + }); + const body = JSON.stringify({ object: 'list', data, model: 'gitnexus-benchmark' }); + counters.responseBytes += Buffer.byteLength(body); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(body); + } catch (error) { + if (counters) counters.error ??= error.message; + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: error.message } })); + } + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + return { + url: `http://127.0.0.1:${server.address().port}/v1`, + begin(failAfter = Infinity) { + active = { + failAfter, + requests: 0, + rejectedRequests: 0, + acceptedInputs: 0, + requestBytes: 0, + responseBytes: 0, + inputs: new Set(), + }; + return active; + }, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} + +function moduleText(moduleIndex, modules, functions, revision = 0) { + const previous = (moduleIndex + modules - 1) % modules; + const lines = [`import { task_${previous}_0 } from './module-${previous}.js';`]; + for (let i = 0; i < functions; i++) { + const dependency = i ? `task_${moduleIndex}_${i - 1}` : `task_${previous}_0`; + lines.push(`/** Compute deterministic module ${moduleIndex}, operation ${i}. */`); + lines.push(`export function task_${moduleIndex}_${i}(value: number): number {`); + lines.push(` if (value <= 0) return ${i + (moduleIndex === 0 && i === 0 ? revision : 0)};`); + lines.push(` return ${dependency}(value - 1) + ${i + 1};`, '}'); + } + return lines.join('\n') + '\n'; +} + +async function fixture(directory, modules, functions) { + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'package.json'), '{"type":"module"}\n'); + for (let i = 0; i < modules; i++) { + await fs.writeFile(path.join(directory, `module-${i}.ts`), moduleText(i, modules, functions)); + } + const git = (...args) => + execFileSync('git', args, { + cwd: directory, + stdio: 'pipe', + env: { + ...process.env, + GIT_AUTHOR_DATE: '2026-01-01T00:00:00Z', + GIT_COMMITTER_DATE: '2026-01-01T00:00:00Z', + }, + }); + git('init', '-q'); + git('config', 'user.name', 'GitNexus benchmark'); + git('config', 'user.email', 'benchmark@example.invalid'); + git('config', 'commit.gpgsign', 'false'); + const hooks = path.join(directory, '.git', 'empty-hooks'); + await fs.mkdir(hooks); + git('config', 'core.hooksPath', hooks); + git('add', '.'); + git('commit', '-qm', 'fixture'); + return git; +} + +function providerSnapshot(counters) { + return { + requests: counters.requests, + rejectedRequests: counters.rejectedRequests, + acceptedInputs: counters.acceptedInputs, + uniqueInputs: counters.inputs.size, + uniqueInputFingerprint: sha([...counters.inputs].sort().join('\n')), + requestBytes: counters.requestBytes, + responseBytes: counters.responseBytes, + error: counters.error, + }; +} + +async function runCase({ + cli, + scenario, + directory, + modules, + functions, + heapMb, + workers, + provider, + sampleMs, + stopRequested, +}) { + const repo = path.join(directory, 'repo'); + const store = path.join(directory, 'store'); + const git = await fixture(repo, modules, functions); + const env = { ...process.env }; + // Isolate GitNexus configuration/registry/credentials; never read a user's real config. + for (const key of Object.keys(env)) if (key.startsWith('GITNEXUS_')) delete env[key]; + delete env.NODE_OPTIONS; + Object.assign(env, { + GITNEXUS_HOME: path.join(directory, 'home'), + GITNEXUS_STORAGE_PATH: store, + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(256 * 1024 * 1024), + GITNEXUS_WORKER_READY_TIMEOUT_MS: '30000', + }); + const embedding = scenario.startsWith('http-'); + if (embedding) + Object.assign(env, { + GITNEXUS_EMBEDDING_URL: provider.url, + GITNEXUS_EMBEDDING_MODEL: 'gitnexus-benchmark', + GITNEXUS_EMBEDDING_DIMS: '256', + GITNEXUS_EMBEDDING_API_KEY: 'benchmark-only', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '16', + }); + const args = [ + cli, + 'analyze', + repo, + '--index-only', + '--memory-budget', + String(heapMb), + '--workers', + String(workers), + ]; + if (embedding) args.push('--embeddings', '0'); + const counters = provider.begin( + scenario === 'http-resume' ? Math.max(16, Math.floor((modules * functions) / 3)) : Infinity, + ); + const invoke = async (name, allowFailure = false) => { + assert( + !stopRequested(), + 'Benchmark interrupted between invocations; native children were not signalled', + ); + const result = await measureProcess(process.execPath, args, { + cwd: repo, + env, + logFile: path.join(directory, `${name}.log`), + sampleMs, + }); + if (!allowFailure) + assert.equal(result.exitCode, 0, `Analyze failed; inspect ${result.logFile}`); + assert(!result.signal, `Analyze ended with ${result.signal}; inspect ${result.logFile}`); + assert(!counters.error, counters.error); + return result; + }; + let setup; + if (['noop', 'incremental', 'http-cached', 'http-resume'].includes(scenario)) { + setup = await invoke('setup', scenario === 'http-resume'); + if (scenario === 'http-resume') { + assert( + counters.rejectedRequests > 0 && counters.acceptedInputs > 0, + 'Resume setup must actually interrupt partially successful work', + ); + counters.failAfter = Infinity; + } + if (scenario === 'incremental') { + await fs.writeFile(path.join(repo, 'module-0.ts'), moduleText(0, modules, functions, 1)); + git('add', 'module-0.ts'); + git('commit', '-qm', 'one-function edit'); + } + } + const beforeMetadata = + setup && scenario !== 'http-resume' + ? JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8')) + : undefined; + const before = providerSnapshot(counters); + const measurement = await invoke('measured'); + const after = providerSnapshot(counters); + if (scenario === 'http-cached') + assert.equal(after.acceptedInputs, before.acceptedInputs, 'Cached run made paid requests'); + const metadata = JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8')); + assertCompletedCase(scenario, metadata, beforeMetadata); + const stats = metadata.stats; + const result = { + ...measurement, + setup, + command: [process.execPath, ...args], + providerBefore: before, + providerAfter: after, + measuredProviderRequests: after.requests - before.requests, + measuredProviderInputs: after.acceptedInputs - before.acceptedInputs, + equalWork: { + files: stats.files, + nodes: stats.nodes, + edges: stats.edges, + embeddings: stats.embeddings ?? 0, + uniqueInputs: after.uniqueInputs, + inputFingerprint: after.uniqueInputFingerprint, + }, + metadata: { + runnerIdentity: metadata.runnerIdentity, + graphGeneration: metadata.graphGeneration, + vectorGeneration: metadata.vectorGeneration, + embeddingCheckpoint: metadata.embeddingCheckpoint, + embeddingJob: metadata.embeddingJob, + }, + }; + await fs.writeFile( + path.join(directory, 'measurement.json'), + JSON.stringify(result, null, 2) + '\n', + ); + return result; +} + +export async function main(argv = process.argv.slice(2)) { + const { values: v } = parseArgs({ + args: argv, + options: { + baseline: { type: 'string' }, + candidate: { type: 'string' }, + output: { type: 'string' }, + scenarios: { type: 'string', default: SCENARIOS.join(',') }, + pairs: { type: 'string', default: '7' }, + warmups: { type: 'string', default: '1' }, + modules: { type: 'string', default: '64' }, + functions: { type: 'string', default: '16' }, + 'heap-mb': { type: 'string', default: '2048' }, + workers: { type: 'string', default: '2' }, + 'sample-ms': { type: 'string', default: '25' }, + 'keep-stores': { type: 'boolean', default: false }, + help: { type: 'boolean' }, + }, + }); + if (v.help) { + console.log(`Usage: node bench/analyze-resume/measure.mjs --baseline /repo/gitnexus/dist/cli/index.js --output /tmp/results.json + --candidate PATH Defaults to this checkout's built CLI + --scenarios LIST ${SCENARIOS.join(',')} + --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only + --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 + --keep-stores Retain owned index/home artifacts (logs and measurements always retained) +Includes full CLI startup, graph/manifest/native writes and final index work. +HTTP resume uses a deterministic provider outage, not an unsafe native-process kill. +RSS is sampled Linux process-family RSS; other platforms report it unavailable. +Local-model, live-query latency, default-budget and platform cases remain separate acceptance requirements.`); + return; + } + assert(v.baseline && v.output, '--baseline and --output are required'); + const numbers = {}; + for (const key of [ + 'pairs', + 'warmups', + 'modules', + 'functions', + 'heap-mb', + 'workers', + 'sample-ms', + ]) { + numbers[key] = Number(v[key]); + assert( + Number.isSafeInteger(numbers[key]) && numbers[key] >= (key === 'warmups' ? 0 : 1), + `Invalid --${key}`, + ); + } + assert(numbers.modules >= 2, '--modules must be at least 2'); + const scenarios = v.scenarios.split(','); + assert( + scenarios.length && + new Set(scenarios).size === scenarios.length && + scenarios.every((x) => SCENARIOS.includes(x)), + 'Invalid or repeated scenario', + ); + const candidate = + v.candidate ?? fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url)); + const clis = { baseline: path.resolve(v.baseline), candidate: path.resolve(candidate) }; + for (const cli of Object.values(clis)) await fs.access(cli); + const runtime = { + baseline: await runtimeEnvironment(clis.baseline), + candidate: await runtimeEnvironment(clis.candidate), + }; + assertComparableRuntimes(runtime.baseline, runtime.candidate); + const output = path.resolve(v.output); + await fs.mkdir(path.dirname(output), { recursive: true }); + // Never overwrite a prior result; evidence files and temporary directories are exclusively owned. + const resultFile = await fs.open(output, 'wx'); + const work = await fs.mkdtemp(path.join(path.dirname(output), 'analyze-pairs-')); + const provider = await startProvider(); + let interrupted = false; + const interrupt = () => { + interrupted = true; + console.error( + 'Stopping between invocations; waiting for the active native CLI to exit normally.', + ); + }; + process.on('SIGINT', interrupt); + const report = { + schemaVersion: 1, + createdAt: new Date().toISOString(), + work, + environment: { + node: process.version, + platform: process.platform, + architecture: process.arch, + cpu: os.cpus()[0]?.model, + availableParallelism: os.availableParallelism(), + totalMemory: os.totalmem(), + settings: numbers, + runtime, + clis: {}, + }, + limitations: [ + 'Synthetic corpus only; stats and deterministic input coverage are checked, not query ranking.', + 'Linux RSS sampling is a lower-bound observation, not a hard memory limit or exact peak.', + 'CPU/disk, phase timings, native reservations and queue peaks are not collected by this harness.', + 'Local model, live query latency, default budgets, Windows and final production integration require additional validation.', + ], + acceptance: 'incomplete', + samples: [], + summaries: {}, + }; + const save = async () => { + const bytes = Buffer.from(JSON.stringify(report, null, 2) + '\n'); + await resultFile.write(bytes, 0, bytes.length, 0); + await resultFile.truncate(bytes.length); + await resultFile.sync(); + }; + try { + for (const [side, cli] of Object.entries(clis)) { + report.environment.clis[side] = { + path: cli, + entryDigest: sha(await fs.readFile(cli)), + commit: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: path.dirname(cli), + encoding: 'utf8', + }).trim(), + }; + } + await save(); + for (const scenario of scenarios) { + const pairs = []; + for (let iteration = -numbers.warmups; iteration < numbers.pairs; iteration++) { + const ordinal = iteration + numbers.warmups; + const order = ordinal % 2 === 0 ? ['baseline', 'candidate'] : ['candidate', 'baseline']; + const pair = { scenario, iteration, warmup: iteration < 0, order }; + report.samples.push(pair); + for (const side of order) { + assert( + !interrupted, + 'Benchmark interrupted between invocations; partial evidence retained', + ); + const directory = path.join(work, `${scenario}-${ordinal}-${side}`); + pair[side] = await runCase({ + cli: clis[side], + scenario, + directory, + modules: numbers.modules, + functions: numbers.functions, + heapMb: numbers['heap-mb'], + workers: numbers.workers, + sampleMs: numbers['sample-ms'], + provider, + stopRequested: () => interrupted, + }); + await save(); + console.log( + JSON.stringify({ + scenario, + iteration, + side, + elapsedMs: pair[side].elapsedMs, + requests: pair[side].measuredProviderRequests, + rss: pair[side].sampledPeakProcessTreeRssBytes, + }), + ); + if (!v['keep-stores']) { + // Only paths created inside this invocation's mkdtemp root are removed. + await fs.rm(path.join(directory, 'store'), { recursive: true, force: true }); + await fs.rm(path.join(directory, 'home'), { recursive: true, force: true }); + } + } + assert.deepEqual( + pair.candidate.equalWork, + pair.baseline.equalWork, + `Unequal work for ${scenario}; inspect retained measurements`, + ); + const baselineDependencies = + pair.baseline.metadata.runnerIdentity?.dependencyRuntime?.digest; + assert( + /^sha256:[a-f0-9]{64}$/.test(baselineDependencies ?? ''), + 'Baseline dependency receipt missing', + ); + assert.equal( + pair.candidate.metadata.runnerIdentity?.dependencyRuntime?.digest, + baselineDependencies, + 'Resolved dependency payloads differ; timings are not comparable', + ); + if (iteration >= 0) + pairs.push({ baseline: pair.baseline.elapsedMs, candidate: pair.candidate.elapsedMs }); + } + report.summaries[scenario] = summarizePairs(pairs); + await save(); + } + console.log( + JSON.stringify( + { output, summaries: report.summaries, acceptance: report.acceptance }, + null, + 2, + ), + ); + } catch (error) { + report.error = error.message; + await save(); + throw error; + } finally { + process.removeListener('SIGINT', interrupt); + await provider.close(); + await resultFile.close(); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/gitnexus/bench/analyze-resume/measure.test.mjs b/gitnexus/bench/analyze-resume/measure.test.mjs new file mode 100644 index 000000000..6eecfbb2a --- /dev/null +++ b/gitnexus/bench/analyze-resume/measure.test.mjs @@ -0,0 +1,127 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { + assertComparableRuntimes, + assertCompletedCase, + measureProcess, + startProvider, + summarizePairs, +} from './measure.mjs'; + +test('incomparable runtime filesystems or dependency bytes cannot produce paired evidence', () => { + const baseline = { + cli: { device: '1', filesystem: 'ext4' }, + nativeEntry: { device: '1', filesystem: 'ext4' }, + lockDigest: 'lock', + nativeEntryDigest: 'native', + }; + assertComparableRuntimes(baseline, structuredClone(baseline)); + const remote = structuredClone(baseline); + remote.cli.device = '2'; + remote.cli.filesystem = 'v9fs'; + assert.throws(() => assertComparableRuntimes(baseline, remote), /different devices/); + assert.throws( + () => assertComparableRuntimes(baseline, { ...baseline, lockDigest: 'other' }), + /lockfiles differ/, + ); + assert.throws( + () => assertComparableRuntimes(baseline, { ...baseline, nativeEntryDigest: 'other' }), + /entry bytes differ/, + ); +}); + +test('completion checks reject a rebuilding no-op and an incomplete successful embedding run', () => { + const metadata = { indexedAt: 'first', stats: { nodes: 10, embeddings: 5 } }; + assertCompletedCase('noop', metadata, metadata); + assert.throws( + () => assertCompletedCase('noop', { ...metadata, indexedAt: 'second' }, metadata), + /No-op changed/, + ); + assert.throws( + () => assertCompletedCase('http-fresh', { ...metadata, embeddingJob: {} }), + /pending/, + ); + assert.throws( + () => assertCompletedCase('http-resume', { ...metadata, embeddingCheckpoint: {} }), + /pending/, + ); + assert.throws( + () => assertCompletedCase('http-fresh', { stats: { nodes: 10, embeddings: 0 } }), + /no vectors/, + ); +}); + +test('paired evidence never treats small samples or equality as acceptance', () => { + assert.equal(summarizePairs([{ baseline: 100, candidate: 90 }]).evidence, 'smoke-only'); + const equal = Array.from({ length: 7 }, () => ({ baseline: 100, candidate: 100 })); + assert.equal(summarizePairs(equal).evidence, 'inconclusive'); + assert.equal(summarizePairs(equal.map((p) => ({ ...p, candidate: 110 }))).evidence, 'regression'); + assert.equal( + summarizePairs(equal.map((p) => ({ ...p, candidate: 90 }))).evidence, + 'no-detected-regression', + ); + assert.throws(() => summarizePairs([{ baseline: 0, candidate: 1 }])); + assert.throws(() => summarizePairs([])); +}); + +test('process measurement waits for completion, records failure and preserves logs', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'analyze-measure-test-')); + try { + const logFile = path.join(directory, 'child.log'); + const result = await measureProcess( + process.execPath, + ['-e', 'setTimeout(() => { console.log("finished"); process.exitCode = 7; }, 70)'], + { cwd: directory, env: process.env, logFile, sampleMs: 10 }, + ); + assert.equal(result.exitCode, 7); + assert.equal(result.signal, null); + assert(result.elapsedMs >= 70); + assert.equal((await fs.readFile(logFile, 'utf8')).trim(), 'finished'); + if (process.platform === 'linux') { + assert(result.rssSamples > 0); + assert(result.sampledPeakProcessTreeRssBytes > 0); + } + await assert.rejects( + () => + measureProcess(process.execPath, ['-e', ''], { + cwd: directory, + env: process.env, + logFile, + }), + { code: 'EEXIST' }, + ); + assert.equal((await fs.readFile(logFile, 'utf8')).trim(), 'finished'); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +}); + +test('provider outage preserves paid-input accounting and deterministic vectors', async () => { + const provider = await startProvider(16); + const request = (input) => + fetch(`${provider.url}/embeddings`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ input }), + }); + try { + const counters = provider.begin(2); + const first = await (await request(['alpha', 'beta'])).json(); + const failure = await request(['gamma']); + assert.equal(failure.status, 503); + await failure.json(); + assert.equal(counters.acceptedInputs, 2); + assert.equal(counters.rejectedRequests, 1); + counters.failAfter = Infinity; + const retry = await (await request(['alpha'])).json(); + assert.deepEqual(retry.data[0].embedding, first.data[0].embedding); + assert.equal(counters.acceptedInputs, 3); + assert.equal(counters.inputs.size, 2); + assert.equal(counters.requests, 3); + } finally { + await provider.close(); + } +}); From 04dc44efa5776d5d649191b7674d2afaae30ab60 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 07:32:47 +0000 Subject: [PATCH 16/67] feat(index): publish logical graph generations Retain private database files across incremental generations and fence prepared publication against superseding writers. Exit-only producers must exit before publication; interrupted in-place writes remain explicitly dirty. --- gitnexus/src/core/analysis-publication.ts | 162 ++++++++++-- gitnexus/src/storage/repo-meta.ts | 5 +- gitnexus/src/storage/vector-generation.ts | 2 + .../test/unit/analysis-publication.test.ts | 242 +++++++++++++++++- gitnexus/test/unit/vector-generation.test.ts | 10 + 5 files changed, 399 insertions(+), 22 deletions(-) diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts index 52f14ab3b..ad0bb14e3 100644 --- a/gitnexus/src/core/analysis-publication.ts +++ b/gitnexus/src/core/analysis-publication.ts @@ -10,6 +10,7 @@ import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/emb const RECEIPT = 'analysis-publication.json'; const GENERATION = /^[a-f0-9]{32}$/; +const PRIVATE_GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/; const DIGEST = /^[a-f0-9]{64}$/; const STAGING = /^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; const MAX_RECEIPT_BYTES = 4096; @@ -17,17 +18,30 @@ const MAX_METADATA_BYTES = 256 * 1024 * 1024; const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); export interface PreparedAnalysisPublication { - version: 1; + version: 2; + mode: 'staged' | 'in-place'; id: string; + /** Source basename; also names the existing file for in-place writes. */ stagingFile: string; graphFile: string; metadataFile: string; metadataBytes: number; metadataDigest: string; + /** Exact prior metadata, or absence, observed under the slot's write lock. */ + priorMetadataDigest: string | null; + /** Permitted intermediate state while retiring a canonical legacy graph. */ + redirectedMetadataDigest?: string; + /** Exit-only callers may not publish while this process is still alive. */ + writerPid?: number; /** A hard-link destination for the previous legacy graph; never copied. */ legacyGraphFile?: string; } +export type AnalysisPublicationOptions = ( + | { mode?: 'staged' } + | { mode: 'in-place'; attemptId: string } +) & { waitForProcessExit?: boolean }; + export interface AnalysisPublicationActions { /** Open read-only and verify graph/schema/FTS against the prepared metadata. * Must close the verification handle before returning (including on Windows). @@ -42,13 +56,23 @@ function validateReceipt(value: unknown): asserts value is PreparedAnalysisPubli const r = value as PreparedAnalysisPublication; if ( !r || - r.version !== 1 || + r.version !== 2 || !GENERATION.test(r.id) || - !STAGING.test(r.stagingFile) || - r.graphFile !== `lbug.generation.${r.id}` || + !PRIVATE_GRAPH.test(r.graphFile) || + (r.mode === 'staged' + ? !STAGING.test(r.stagingFile) || r.graphFile !== `lbug.generation.${r.id}` + : r.mode !== 'in-place' || + (r.stagingFile === 'lbug' + ? r.graphFile !== `lbug.generation.${r.id}` + : !PRIVATE_GRAPH.test(r.stagingFile) || r.graphFile !== r.stagingFile)) || r.metadataFile !== `analysis-meta-${r.id}.json` || + (r.priorMetadataDigest !== null && !DIGEST.test(r.priorMetadataDigest)) || + (r.redirectedMetadataDigest !== undefined && + (!r.legacyGraphFile || !DIGEST.test(r.redirectedMetadataDigest))) || + (r.writerPid !== undefined && (!Number.isSafeInteger(r.writerPid) || r.writerPid < 1)) || (r.legacyGraphFile !== undefined && - (!/^lbug\.generation\.[a-f0-9]{32}$/.test(r.legacyGraphFile) || + (r.mode !== 'staged' || + !PRIVATE_GRAPH.test(r.legacyGraphFile) || r.legacyGraphFile === r.graphFile)) || !Number.isSafeInteger(r.metadataBytes) || r.metadataBytes < 1 || @@ -109,6 +133,58 @@ async function writeDurable(directory: string, name: string, bytes: Buffer): Pro } } +async function readSlotMetadata(root: string): Promise { + try { + const bytes = await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES); + const metadata: RepoMeta = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + if (!metadata || typeof metadata.repoPath !== 'string') + throw new Error('Invalid graph slot metadata; artifacts retained'); + return metadata; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +const metadataDigest = (metadata: RepoMeta | null): string | null => + metadata === null ? null : hash(JSON.stringify(metadata)); + +/** Replaces the existing pre-write dirty stamp, never an extra post-write stamp. + * The coordinator holds the slot lock and must await this before mutating native + * data. An interrupted in-place attempt requires the usual dirty-graph rebuild. + */ +export async function beginInPlaceAnalysis( + directory: string, + metadata: RepoMeta, + saveMetadata: AnalysisPublicationActions['saveMetadata'], +): Promise { + const root = await embeddingJobDirectory(directory); + if ( + !metadata.incrementalInProgress || + (metadata.graphPath !== undefined && !PRIVATE_GRAPH.test(metadata.graphPath)) + ) + throw new Error('In-place publication requires a private graph and a dirty write plan'); + if (await readPreparedAnalysis(root)) + throw new Error('Recover the pending graph publication before starting another'); + // Refuse an aliased native target before the caller can perform its first write. + await regularFile(path.join(root, metadata.graphPath ?? 'lbug')); + const prior = await readSlotMetadata(root); + if (prior && prior.repoPath !== metadata.repoPath) + throw new Error('Graph slot belongs to another repository'); + const dirty: RepoMeta = { + ...metadata, + incrementalInProgress: { + ...metadata.incrementalInProgress, + writePlan: 'in-place', + attemptId: randomUUID().replaceAll('-', ''), + }, + }; + delete dirty.graphGeneration; + await saveMetadata(dirty); + await syncPublishedMetadata(root, dirty); + return dirty; +} + /** Call after graph/FTS verification and a successful native checkpoint, before * native close. A close crash leaves the owned build and its publication intent. * No model or vector work may start until publishPreparedAnalysis completes. @@ -118,22 +194,43 @@ export async function prepareAnalysisPublication( buildPath: string, metadata: RepoMeta, coveredContent: string, + options: AnalysisPublicationOptions = {}, ): Promise { const root = await embeddingJobDirectory(directory); if ( !DIGEST.test(coveredContent) || !metadata.repoPath || !metadata.runnerIdentity || - metadata.incrementalInProgress + metadata.incrementalInProgress || + metadata.graphWriteCollapsed ) throw new Error('Graph publication requires verified source and analyzer metadata'); - if (path.dirname(path.resolve(buildPath)) !== root || !STAGING.test(path.basename(buildPath))) + const mode = options.mode ?? 'staged'; + const sourceFile = path.basename(buildPath); + if ( + path.dirname(path.resolve(buildPath)) !== root || + (mode === 'staged' + ? !STAGING.test(sourceFile) + : sourceFile !== 'lbug' && !PRIVATE_GRAPH.test(sourceFile)) + ) throw new Error('Graph build is outside its owned staging directory'); if (!(await regularFile(buildPath))) throw new Error('Graph build is missing'); if (await regularFile(path.join(root, RECEIPT))) throw new Error('Recover the pending graph publication before preparing another'); - const id = randomUUID().replaceAll('-', ''); - const graphFile = `lbug.generation.${id}`; + const prior = await readSlotMetadata(root); + if (prior && prior.repoPath !== metadata.repoPath) + throw new Error('Graph slot belongs to another repository'); + if ( + options.mode === 'in-place' && + (!GENERATION.test(options.attemptId) || + prior?.incrementalInProgress?.attemptId !== options.attemptId || + (prior.graphPath ?? 'lbug') !== sourceFile || + prior.graphGeneration) + ) + throw new Error('In-place graph attempt is missing or superseded; artifacts retained'); + const id = options.mode === 'in-place' ? options.attemptId : randomUUID().replaceAll('-', ''); + const graphFile = + mode === 'in-place' && sourceFile !== 'lbug' ? sourceFile : `lbug.generation.${id}`; const metadataFile = `analysis-meta-${id}.json`; const preparedMeta: RepoMeta = { ...metadata, @@ -144,17 +241,24 @@ export async function prepareAnalysisPublication( if (bytes.length > MAX_METADATA_BYTES) throw new Error('Graph metadata exceeds the supported publication size'); const receipt: PreparedAnalysisPublication = { - version: 1, + version: 2, + mode, id, stagingFile: path.basename(buildPath), graphFile, metadataFile, metadataBytes: bytes.length, metadataDigest: hash(bytes), - ...((await regularFile(path.join(root, 'lbug'))) + priorMetadataDigest: metadataDigest(prior), + ...(options.waitForProcessExit ? { writerPid: process.pid } : {}), + ...(mode === 'staged' && (await regularFile(path.join(root, 'lbug'))) ? { legacyGraphFile: `lbug.generation.${randomUUID().replaceAll('-', '')}` } : {}), }; + if (receipt.legacyGraphFile && prior && !prior.graphPath) + receipt.redirectedMetadataDigest = hash( + JSON.stringify({ ...prior, graphPath: receipt.legacyGraphFile }), + ); validateReceipt(receipt); await writeDurable(root, metadataFile, bytes); const payload = JSON.stringify(receipt); @@ -221,6 +325,7 @@ async function retireLegacyGraph( receipt: PreparedAnalysisPublication, metadata: RepoMeta, actions: AnalysisPublicationActions, + previous: RepoMeta | null, ): Promise { const legacy = path.join(root, 'lbug'); if (!receipt.legacyGraphFile) { @@ -241,12 +346,9 @@ async function retireLegacyGraph( await fs.link(legacy, retired); await syncEmbeddingJobDirectory(root); } - const previous = JSON.parse( - (await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES)).toString('utf8'), - ) as RepoMeta; - if (previous.repoPath !== metadata.repoPath) + if (previous && previous.repoPath !== metadata.repoPath) throw new Error('Legacy graph belongs to another repository'); - if (!previous.graphPath) { + if (previous && !previous.graphPath) { const redirected = { ...previous, graphPath: receipt.legacyGraphFile }; await actions.saveMetadata(redirected); await syncPublishedMetadata(root, redirected); @@ -257,7 +359,8 @@ async function retireLegacyGraph( /** Also recovers crashes after rename or metadata publication. Every attempt * re-verifies the owned native artifact; a generation identifier is not proof. - * Existing graph files remain untouched so readers can finish their snapshots. + * Staged writes retain the prior reader artifact. In-place writes have already + * invalidated it and publish a new logical generation on the same private file. */ export async function publishPreparedAnalysis( directory: string, @@ -266,6 +369,14 @@ export async function publishPreparedAnalysis( const root = await embeddingJobDirectory(directory); const receipt = await readPreparedAnalysis(root); if (!receipt) return undefined; + if (receipt.writerPid !== undefined) { + try { + process.kill(receipt.writerPid, 0); + throw new Error('Graph writer has not exited; publication must wait for physical release'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } + } const bytes = await boundedRead(path.join(root, receipt.metadataFile), receipt.metadataBytes); if (bytes.length !== receipt.metadataBytes || hash(bytes) !== receipt.metadataDigest) throw new Error('Prepared graph metadata checksum mismatch; artifacts retained'); @@ -277,19 +388,30 @@ export async function publishPreparedAnalysis( !DIGEST.test(metadata.graphGeneration.coveredContent) ) throw new Error('Prepared graph generation mismatch'); + const previous = await readSlotMetadata(root); + const currentDigest = metadataDigest(previous); + if ( + currentDigest !== receipt.priorMetadataDigest && + currentDigest !== receipt.redirectedMetadataDigest && + currentDigest !== receipt.metadataDigest + ) + throw new Error( + 'Prepared graph publication was superseded; artifacts retained; retry analysis', + ); const staged = path.join(root, receipt.stagingFile); const published = path.join(root, receipt.graphFile); const stagedExists = await regularFile(staged); const publishedExists = await regularFile(published); - if (stagedExists === publishedExists) + const retainedInPlace = staged === published; + if (retainedInPlace ? !publishedExists : stagedExists === publishedExists) throw new Error('Prepared publication has missing or conflicting graph artifacts'); const source = stagedExists ? staged : published; await actions.verify(source, metadata); - if (stagedExists) { + if (stagedExists && !retainedInPlace) { await fs.rename(staged, published); await syncEmbeddingJobDirectory(root); } - await retireLegacyGraph(root, receipt, metadata, actions); + await retireLegacyGraph(root, receipt, metadata, actions, previous); await actions.saveMetadata(metadata); // The canonical pointer must be durable before callers start paid work. await syncPublishedMetadata(root, metadata); diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 594599cf6..66d9cc54c 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -119,7 +119,8 @@ export interface RepoMeta { * validated by `resolveGraphPath`. Absent means legacy `/lbug`. */ graphPath?: string; - /** A verified immutable graph, published independently of optional vectors. */ + /** A verified logical generation, published independently of optional vectors. + * Private incremental writes invalidate this receipt before mutating its file. */ graphGeneration?: { version: 1; id: string; @@ -480,6 +481,8 @@ export interface RepoMeta { * to a known-good index. */ incrementalInProgress?: { + /** Unique ownership fence for a prepared in-place graph publication. */ + attemptId?: string; /** When the run started (epoch ms). */ startedAt: number; /** Last dirty-flag refresh (epoch ms). */ diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index 15f29de71..2d8d78b66 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -36,6 +36,8 @@ export function selectVectorGeneration( graphPath: string, exactScanLimit: number, ): VectorReadSelection { + if (metadata?.incrementalInProgress || metadata?.graphWriteCollapsed) + return { kind: 'unavailable', reason: 'graph-generation-unverified' }; if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob) return { kind: 'legacy' }; const graph = metadata?.graphGeneration; diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts index bf474455f..e2e94840e 100644 --- a/gitnexus/test/unit/analysis-publication.test.ts +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -2,13 +2,15 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { + beginInPlaceAnalysis, prepareAnalysisPublication, publishPreparedAnalysis, readPreparedAnalysis, } from '../../src/core/analysis-publication.js'; -import { getStoragePaths, saveMeta } from '../../src/storage/repo-manager.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import { resolveGraphPath } from '../../src/storage/shared-store.js'; import type { RepoMeta } from '../../src/storage/repo-meta.js'; @@ -41,6 +43,92 @@ afterEach(async () => { }); describe('durable graph publication', () => { + it('publishes only after the real preparing process exits', async () => { + const { root, stage, actions } = await fixture(); + const moduleUrl = new URL('../../dist/core/analysis-publication.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { prepareAnalysisPublication } from ${JSON.stringify(moduleUrl)}; + process.once('message', async ({ root, stage, metadata, content }) => { + await prepareAnalysisPublication(root, stage, metadata, content, { waitForProcessExit: true }); + process.once('message', () => process.disconnect()); + process.send('prepared'); + }); + `, + ], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ); + const exited = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', resolve); + }); + const prepared = new Promise((resolve) => child.once('message', resolve)); + try { + child.send({ root, stage, metadata, content }); + await Promise.race([ + prepared, + exited.then(() => { + throw new Error('Writer exited before preparation'); + }), + ]); + expect(child.exitCode).toBeNull(); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('has not exited'); + expect(actions.verify).not.toHaveBeenCalled(); + child.send('release'); + expect(await exited).toBe(0); + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration).toBeDefined(); + expect(actions.verify).toHaveBeenCalledOnce(); + } finally { + if (child.connected) child.send('release'); + await exited; + } + }); + + it('waits for an exit-only writer even after it has prepared the graph', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + expect(receipt.writerPid).toBe(process.pid); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('has not exited'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + const probe = vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('exited'), { code: 'ESRCH' }); + }); + await publishPreparedAnalysis(root, actions); + expect(probe).toHaveBeenCalledWith(process.pid, 0); + expect(actions.verify).toHaveBeenCalledOnce(); + }); + + it('retains a prepared graph when process liveness cannot be established', async () => { + const { root, stage, actions } = await fixture(); + await prepareAnalysisPublication(root, stage, metadata, content, { waitForProcessExit: true }); + vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('permission denied'), { code: 'EPERM' }); + }); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('permission denied'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + }); + + it('rejects a superseded attempt before verifying or moving its graph', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const newer = { ...metadata, lastCommit: 'newer-writer' }; + await saveMeta(root, newer); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('superseded'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await loadMeta(root)).toEqual(newer); + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + }); + it('publishes a verified generation and keeps the previous reader artifact', async () => { const { root, stage, actions } = await fixture(); const receipt = await prepareAnalysisPublication(root, stage, metadata, content); @@ -182,3 +270,155 @@ describe('durable graph publication', () => { expect(() => getStoragePaths('/repo', 'feature', root)).toThrow('regular file'); }); }); + +describe('logical in-place generations', () => { + it('invalidates readiness before writes and retains one physical file across generations', async () => { + const { root, actions } = await fixture(); + let previous = metadata; + let file = path.join(root, 'lbug'); + const before = await fs.stat(file); + const generations: string[] = []; + for (let iteration = 0; iteration < 3; iteration++) { + const dirty = await beginInPlaceAnalysis( + root, + { + ...previous, + incrementalInProgress: { startedAt: Date.now(), toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + expect((await loadMeta(root))?.graphGeneration).toBeUndefined(); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + expect((await loadMeta(root))?.incrementalInProgress?.attemptId).toBe(attemptId); + // Simulate the delta write only after the dirty stamp is durable. + await fs.writeFile(file, 'verified graph'); + const receipt = await prepareAnalysisPublication(root, file, previous, content, { + mode: 'in-place', + attemptId, + }); + const published = await publishPreparedAnalysis(root, actions); + if (!published) throw new Error('Expected a published generation'); + file = resolveGraphPath(root); + expect((await fs.stat(file)).ino).toBe(before.ino); + expect(receipt.legacyGraphFile).toBeUndefined(); + expect(published.incrementalInProgress).toBeUndefined(); + expect(published.graphGeneration?.id).toBe(attemptId); + generations.push(attemptId); + previous = published; + } + expect(new Set(generations).size).toBe(3); + expect( + (await fs.readdir(root)).filter((name) => /^lbug\.generation\.[a-f0-9]{32}$/.test(name)), + ).toHaveLength(1); + await expect(fs.stat(path.join(root, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.each(['before-rename', 'after-rename', 'after-metadata'] as const)( + 'recovers in-place publication %s without restoring a false old generation', + async (point) => { + const { root, actions } = await fixture(); + const dirty = await beginInPlaceAnalysis( + root, + { + ...metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + await fs.writeFile(path.join(root, 'lbug'), 'verified graph'); + const receipt = await prepareAnalysisPublication( + root, + path.join(root, 'lbug'), + metadata, + content, + { + mode: 'in-place', + attemptId, + }, + ); + await expect( + publishPreparedAnalysis(root, { + verify: + point === 'before-rename' + ? async () => { + throw new Error('crash'); + } + : actions.verify, + saveMetadata: async (meta) => { + if (point === 'after-metadata') await actions.saveMetadata(meta); + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + if (point !== 'after-metadata') { + expect((await loadMeta(root))?.graphGeneration).toBeUndefined(); + expect((await loadMeta(root))?.incrementalInProgress?.attemptId).toBe(attemptId); + } + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration?.id).toBe(attemptId); + expect(published?.incrementalInProgress).toBeUndefined(); + }, + ); + + it('requires a durable matching attempt and rejects another writer before preparation', async () => { + const { root, actions } = await fixture(); + const file = path.join(root, 'lbug'); + await expect( + prepareAnalysisPublication(root, file, metadata, content, { + mode: 'in-place', + attemptId: 'a'.repeat(32), + }), + ).rejects.toThrow('missing or superseded'); + const dirty = await beginInPlaceAnalysis( + root, + { + ...metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + await saveMeta(root, { ...metadata, lastCommit: 'newer' }); + await expect( + prepareAnalysisPublication(root, file, metadata, content, { + mode: 'in-place', + attemptId, + }), + ).rejects.toThrow('missing or superseded'); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + }); + + it('rejects a shared pointer, missing dirty plan and an outstanding prepared publication', async () => { + const { root, stage, actions } = await fixture(); + await expect(beginInPlaceAnalysis(root, metadata, actions.saveMetadata)).rejects.toThrow( + 'dirty', + ); + const dirty = { ...metadata, incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }; + await expect( + beginInPlaceAnalysis(root, { ...dirty, graphPath: '/shared/lbug' }, actions.saveMetadata), + ).rejects.toThrow('private'); + await prepareAnalysisPublication(root, stage, metadata, content); + await expect(beginInPlaceAnalysis(root, dirty, actions.saveMetadata)).rejects.toThrow( + 'Recover', + ); + }); + + it('rejects an aliased target or foreign slot before changing its dirty marker', async () => { + const { root, stage, actions } = await fixture(); + const dirty = { ...metadata, incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }; + await expect( + beginInPlaceAnalysis(root, { ...dirty, repoPath: '/other-repo' }, actions.saveMetadata), + ).rejects.toThrow('another repository'); + await fs.unlink(path.join(root, 'lbug')); + await fs.symlink(stage, path.join(root, 'lbug')); + await expect(beginInPlaceAnalysis(root, dirty, actions.saveMetadata)).rejects.toThrow( + 'regular file', + ); + expect(await loadMeta(root)).toEqual(metadata); + }); +}); diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts index 9b1947368..6ccb2f367 100644 --- a/gitnexus/test/unit/vector-generation.test.ts +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -38,6 +38,16 @@ afterEach(async () => { }); describe('vector generation routing', () => { + it.each([ + { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }, + { graphWriteCollapsed: { expected: 100, persisted: 1 } }, + ])('gates incomplete legacy and generation graphs before vector access: %o', (incomplete) => { + for (const base of [meta, { repoPath: '/repo', lastCommit: '', indexedAt: '' }]) + expect(select({ ...base, ...incomplete })).toEqual({ + kind: 'unavailable', + reason: 'graph-generation-unverified', + }); + }); it('preserves the legacy reader and routes a complete matching artifact', () => { expect(select({ repoPath: '/repo', lastCommit: '', indexedAt: '' })).toEqual({ kind: 'legacy', From 7999918fec589c6d1d472082f336b8db35070068 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 08:02:05 +0000 Subject: [PATCH 17/67] feat(index): guard graph process handoff Wait for actual process exit and reacquire the ordinary slot lock before publishing the prepared attempt. Recheck placement and the registered name, and retain graph discoverability after cancellation. Add a bounded nonterminal worker receipt without serializing the live graph. Production phase activation remains separate. --- .../src/core/analysis-phase-supervisor.ts | 173 ++++++++++++ .../core/auto-sync/analysis-worker-launch.ts | 2 +- gitnexus/src/core/run-analyze.ts | 3 + gitnexus/src/server/analyze-worker-core.ts | 17 ++ .../src/server/analyze-worker-protocol.ts | 10 +- gitnexus/src/server/analyze-worker.ts | 1 + .../unit/analysis-phase-supervisor.test.ts | 260 ++++++++++++++++++ .../test/unit/analyze-worker-core.test.ts | 88 ++++++ .../unit/auto-sync-analysis-worker.test.ts | 16 ++ 9 files changed, 568 insertions(+), 2 deletions(-) create mode 100644 gitnexus/src/core/analysis-phase-supervisor.ts create mode 100644 gitnexus/test/unit/analysis-phase-supervisor.test.ts diff --git a/gitnexus/src/core/analysis-phase-supervisor.ts b/gitnexus/src/core/analysis-phase-supervisor.ts new file mode 100644 index 000000000..66dd46b66 --- /dev/null +++ b/gitnexus/src/core/analysis-phase-supervisor.ts @@ -0,0 +1,173 @@ +/** Filesystem-only handoff from an exit-only graph writer. No model/native imports. */ +import path from 'node:path'; +import type { ChildProcess } from 'node:child_process'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { embeddingJobDirectory } from '../storage/embedding-job.js'; +import { + publishPreparedAnalysis, + readPreparedAnalysis, + type AnalysisPublicationActions, +} from './analysis-publication.js'; + +export interface AnalysisPhaseTarget { + repoPath: string; + storagePath: string; + metadataDirectory: string; + repoName: string; + commit: string; +} + +/** Only this bounded receipt crosses IPC; the live graph stays in its child. */ +export interface PreparedAnalysisHandoff extends AnalysisPhaseTarget { + version: 1; + publicationId: string; +} + +export function projectPreparedAnalysisHandoff(value: unknown): PreparedAnalysisHandoff { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid prepared analysis handoff'); + const input = value as Record; + const text = (name: string, maximum: number, allowEmpty = false): string => { + const value = input[name]; + if (typeof value !== 'string' || (!allowEmpty && !value.length) || value.length > maximum) + throw new Error(`Invalid prepared analysis ${name}`); + return value; + }; + const projected: PreparedAnalysisHandoff = { + version: 1, + publicationId: text('publicationId', 32), + repoPath: text('repoPath', 32768), + storagePath: text('storagePath', 32768), + metadataDirectory: text('metadataDirectory', 32768), + repoName: text('repoName', 1024), + commit: text('commit', 64, true), + }; + if ( + input.version !== 1 || + !/^[a-f0-9]{32}$/.test(projected.publicationId) || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})?$/.test(projected.commit) || + ![projected.repoPath, projected.storagePath, projected.metadataDirectory].every(path.isAbsolute) + ) + throw new Error('Invalid prepared analysis identity or location'); + return projected; +} + +/** Call when the process is spawned. A prepared/complete IPC is never release + * evidence. Errors reject without authorizing any continuation or killing the + * child; its existing launcher retains cancellation/abandonment ownership. */ +export function waitForAnalysisProcessExit(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + child.removeListener('exit', onExit); + child.removeListener('error', onError); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + if (code === 0 && signal === null) resolve(); + else + reject( + new Error( + `Graph process exited unsuccessfully (${signal ?? code ?? 'unknown'}); prepared artifacts retained`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + if (child.exitCode !== null || child.signalCode !== null) { + onExit(child.exitCode, child.signalCode); + return; + } + child.once('exit', onExit); + child.once('error', onError); + }); +} + +export interface AnalysisPhasePublicationActions extends AnalysisPublicationActions { + /** Re-resolve checkout placement, current HEAD and the registered name. */ + resolveTarget: () => Promise; + /** Finish registry/shared-store/cache bookkeeping and reader refresh. Called + * under the ordinary slot lock after native verification and publication. + * A cancellation after publication must still leave this graph available. */ + finalizeGraph: (metadata: RepoMeta) => Promise; + signal?: AbortSignal; + log?: (message: string) => void; + onWaitStart?: () => void; +} + +function requireSameTarget(expected: AnalysisPhaseTarget, actual: AnalysisPhaseTarget): void { + if ( + expected.repoName !== actual.repoName || + expected.commit !== actual.commit || + (['repoPath', 'storagePath', 'metadataDirectory'] as const).some( + (key) => path.resolve(expected[key]) !== path.resolve(actual[key]), + ) + ) + throw new Error( + 'Analysis slot, checkout or registered name changed during the graph handoff; retry before embedding', + ); +} + +/** The launcher calls this only after its actual-exit promise resolves. The + * persisted writer-PID guard independently rejects an early or stale handoff. + * No borrowed lock: acquire normally, re-resolve under ownership, then finish + * the exact prepared attempt. An embedding continuation must acquire its own + * lock and recheck this returned generation before making paid requests. */ +export async function publishAnalysisAfterProcessExit( + handoff: PreparedAnalysisHandoff, + writerPid: number, + actions: AnalysisPhasePublicationActions, +): Promise { + const expected = projectPreparedAnalysisHandoff(handoff); + if (!Number.isSafeInteger(writerPid) || writerPid < 1) + throw new Error('Missing graph writer PID'); + actions.signal?.throwIfAborted(); + requireSameTarget(expected, await actions.resolveTarget()); + const root = await embeddingJobDirectory(expected.metadataDirectory); + const lock = await acquireIndexLock(root, { + sweep: false, + log: actions.log, + onWaitStart: actions.onWaitStart, + }); + try { + requireExclusiveIndexLock( + lock, + 'Graph publication requires exclusive ownership of its index slot', + ); + actions.signal?.throwIfAborted(); + requireSameTarget(expected, await actions.resolveTarget()); + const prepared = await readPreparedAnalysis(root); + if (prepared?.id !== expected.publicationId || prepared.writerPid !== writerPid) + throw new Error( + 'Prepared graph attempt changed during handoff; artifacts retained; retry analysis', + ); + const metadata = await publishPreparedAnalysis(root, { + saveMetadata: actions.saveMetadata, + verify: async (file, metadata) => { + if (metadata.repoPath !== expected.repoPath || metadata.lastCommit !== expected.commit) + throw new Error('Prepared graph belongs to a different checkout'); + actions.signal?.throwIfAborted(); + await actions.verify(file, metadata); + actions.signal?.throwIfAborted(); + }, + }); + if (!metadata) throw new Error('Prepared graph disappeared during publication'); + // Once publication starts, complete discoverability even if cancellation + // arrived during its filesystem operations. Paid work remains inadmissible. + const finalized = await actions.finalizeGraph(metadata); + requireSameTarget(expected, await actions.resolveTarget()); + if ( + finalized.graphGeneration?.id !== expected.publicationId || + finalized.repoPath !== expected.repoPath || + finalized.incrementalInProgress || + finalized.graphWriteCollapsed + ) + throw new Error('Graph finalization did not retain the published generation'); + actions.signal?.throwIfAborted(); + return finalized; + } finally { + lock.release(); + } +} diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 9216754ca..077e74891 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -195,7 +195,7 @@ export function createAutoSyncAnalysisRunner( child.on('message', (message: WorkerMessage) => { // Once timeout/cancellation requested shutdown, its reason owns the // result. A terminal IPC can already be queued behind cancellation. - if (terminalOutcome || terminationError) return; + if (message.type === 'graph-prepared' || terminalOutcome || terminationError) return; if (message.type === 'progress') { // The worker also sends its log lines as progress with percent -1; // those are not progress. A listener's failure must not decide the diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index fb36a9d6c..85eedc469 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -65,6 +65,7 @@ import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/und import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import { summarizeUnresolvedOverloads } from './ingestion/scope-resolution/unresolved-overloads.js'; import type { KnowledgeGraph } from './graph/types.js'; +import type { PreparedAnalysisHandoff } from './analysis-phase-supervisor.js'; import { reconcileGraphNodeIdentities } from './incremental/write-reconciliation.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { @@ -797,6 +798,8 @@ export function analyzeFailureMayHaveMutatedLiveIndex(error: unknown): boolean { } export interface AnalyzeResult { + /** Exit-only graph phase awaiting its supervisor's verified publication. */ + preparedGraph?: PreparedAnalysisHandoff; repoName: string; repoPath: string; /** The exact storage slot selected and validated for this analysis. */ diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index ddda81a83..b0e38405b 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -20,6 +20,7 @@ import type { AnalyzeOptions } from '../core/run-analyze.js'; import type { WorkerMessage } from './analyze-worker-protocol.js'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; +import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; // Value import (instanceof): index-lock is a lightweight storage primitive // (node:fs/net/crypto only), so this does NOT pull in run-analyze/repo-manager. import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index-lock.js'; @@ -35,6 +36,9 @@ export interface WorkerAnalysisDeps { * both report a terminal outcome (#2264 P3). See {@link createTerminalClaim}. */ claimTerminal: () => boolean; + /** Required for nonterminal graph handoff; never claim the final outcome just + * to test whether cancellation already owns it. */ + isCancelled?: () => boolean; } /** @@ -67,6 +71,19 @@ export async function runWorkerAnalysis( }, ...bootstrapArgs, ); + if (result.preparedGraph) { + if (!deps.isCancelled) throw new Error('Graph handoff requires cancellation state'); + if (result.graphWriteCollapsed) + throw new Error('Collapsed graph cannot be prepared for publication'); + if (!deps.isCancelled()) { + deps.send({ + type: 'graph-prepared', + handoff: projectPreparedAnalysisHandoff(result.preparedGraph), + result: projectAnalyzeResultForIpc(result), + }); + } + return; + } // P2 (#2264): a half-finalized repo — meta.json written but the global // registry entry missing (e.g. a prior collision-aborted run, or a wiped // registry) — must NOT be reported as a successful analysis. Mirror the CLI's diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 7b046bbe3..9beb534e9 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -25,6 +25,7 @@ */ import type { AnalyzeOptions } from '../core/run-analyze.js'; import type { AnalyzeResultIpc } from './analyze-worker-ipc.js'; +import type { PreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; /** Parent → child: start one analysis run. */ export interface StartMessage { @@ -54,6 +55,13 @@ export interface CompleteMessage { result: AnalyzeResultIpc; } +/** Nonterminal: the graph process still owns its native allocations. */ +export interface GraphPreparedMessage { + type: 'graph-prepared'; + handoff: PreparedAnalysisHandoff; + result: AnalyzeResultIpc; +} + export interface ErrorMessage { type: 'error'; message: string; @@ -75,4 +83,4 @@ export interface ErrorMessage { } /** Child → parent IPC messages. Shared with the parent-side launcher. */ -export type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage; +export type WorkerMessage = ProgressMessage | GraphPreparedMessage | CompleteMessage | ErrorMessage; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index 766cf0022..e83b853e1 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -144,6 +144,7 @@ process.on('message', async (msg: ParentMessage) => { assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized, send, claimTerminal, + isCancelled: () => cancellationRequested, }, prepared.runnerIdentity, ); diff --git a/gitnexus/test/unit/analysis-phase-supervisor.test.ts b/gitnexus/test/unit/analysis-phase-supervisor.test.ts new file mode 100644 index 000000000..287661eed --- /dev/null +++ b/gitnexus/test/unit/analysis-phase-supervisor.test.ts @@ -0,0 +1,260 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + projectPreparedAnalysisHandoff, + publishAnalysisAfterProcessExit, + waitForAnalysisProcessExit, + type AnalysisPhasePublicationActions, + type PreparedAnalysisHandoff, +} from '../../src/core/analysis-phase-supervisor.js'; +import { readPreparedAnalysis } from '../../src/core/analysis-publication.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const roots: string[] = []; +const children: { child: ChildProcess; exited: Promise }[] = []; +afterEach(async () => { + for (const { child, exited } of children.splice(0)) { + if (child.connected) child.send({ exitCode: 0 }); + await exited.catch(() => {}); + } + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-phase-'))); + roots.push(root); + const repoPath = path.join(root, 'repo'); + await fs.mkdir(repoPath); + const commit = 'a'.repeat(40); + const metadata: RepoMeta = { + repoPath, + lastCommit: commit, + indexedAt: '2026-11-04T00:00:00Z', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }; + await saveMeta(root, metadata); + await fs.writeFile(path.join(root, 'lbug'), 'old graph'); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'new graph'); + const target = { + repoPath, + storagePath: root, + metadataDirectory: root, + repoName: 'fixture', + commit, + }; + const moduleUrl = new URL('../../dist/core/analysis-publication.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { prepareAnalysisPublication } from ${JSON.stringify(moduleUrl)}; + process.once('message', async ({ root, stage, metadata, target }) => { + const receipt = await prepareAnalysisPublication(root, stage, metadata, 'b'.repeat(64), { waitForProcessExit: true }); + process.once('message', ({ exitCode }) => { process.exitCode = exitCode; process.disconnect(); }); + process.send({ ...target, version: 1, publicationId: receipt.id }); + }); + `, + ], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ); + const exited = waitForAnalysisProcessExit(child); + // Keep rejection observed even when an assertion fails before awaiting it. + void exited.catch(() => {}); + children.push({ child, exited }); + const writerPid = child.pid; + if (writerPid === undefined) throw new Error('Graph fixture did not spawn'); + const prepared = new Promise((resolve) => + child.once('message', (value) => { + resolve(projectPreparedAnalysisHandoff(value)); + }), + ); + child.send({ root, stage, metadata, target }); + const handoff = await Promise.race([ + prepared, + exited.then(() => { + throw new Error('Writer exited before preparation'); + }), + ]); + const actions: AnalysisPhasePublicationActions = { + resolveTarget: vi.fn(async () => ({ ...target })), + saveMetadata: (value) => saveMeta(root, value), + verify: vi.fn(async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('new graph'); + }), + finalizeGraph: vi.fn(async (value) => value), + }; + const release = (exitCode = 0) => child.send({ exitCode }); + const finish = async () => { + await exited; + return publishAnalysisAfterProcessExit(handoff, writerPid, actions); + }; + return { root, child, writerPid, exited, handoff, metadata, target, actions, release, finish }; +} + +describe('graph process handoff', () => { + it('keeps prepared nonterminal and publishes only after actual exit', async () => { + const f = await fixture(); + const provider = vi.fn(); + const finishing = f.finish().then((metadata) => { + provider(); + return metadata; + }); + await new Promise(setImmediate); + expect(f.child.exitCode).toBeNull(); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + f.release(); + const published = await finishing; + expect(published.graphGeneration?.id).toBe(f.handoff.publicationId); + expect(f.actions.finalizeGraph).toHaveBeenCalledOnce(); + expect(f.actions.resolveTarget).toHaveBeenCalledTimes(3); + expect(provider).toHaveBeenCalledOnce(); + }); + + it('rejects an early caller independently of its process-exit waiter', async () => { + const f = await fixture(); + await expect( + publishAnalysisAfterProcessExit(f.handoff, f.writerPid, f.actions), + ).rejects.toThrow('has not exited'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it('retains artifacts when the child exits unsuccessfully after preparing', async () => { + const f = await fixture(); + const finishing = f.finish(); + f.release(1); + await expect(finishing).rejects.toThrow('unsuccessfully'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it('waits for exit after cancellation and never admits a new phase', async () => { + const f = await fixture(); + const controller = new AbortController(); + f.actions.signal = controller.signal; + let settled = false; + const finishing = f.finish().finally(() => { + settled = true; + }); + controller.abort(new Error('cancelled')); + await new Promise(setImmediate); + expect(settled).toBe(false); + expect(f.child.exitCode).toBeNull(); + f.release(); + await expect(finishing).rejects.toThrow('cancelled'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it.each(['repoName', 'metadataDirectory', 'storagePath', 'commit'] as const)( + 'rechecks %s after waiting for the existing slot lock', + async (field) => { + const f = await fixture(); + const held = await acquireIndexLock(f.root, { sweep: false }); + let startedWaiting!: () => void; + const waiting = new Promise((resolve) => { + startedWaiting = resolve; + }); + f.actions.onWaitStart = startedWaiting; + const finishing = f.finish(); + try { + f.release(); + await waiting; + f.actions.resolveTarget = async () => ({ + ...f.target, + [field]: f.target[field] + '-changed', + }); + } finally { + held.release(); + } + await expect(finishing).rejects.toThrow('changed during'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }, + ); + + it('rejects a superseding writer even when the target and commit did not change', async () => { + const f = await fixture(); + f.release(); + await f.exited; + await saveMeta(f.root, { ...f.metadata, indexedAt: 'newer writer' }); + await expect(f.finish()).rejects.toThrow('superseded'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect((await loadMeta(f.root))?.indexedAt).toBe('newer writer'); + }); + + it('rejects a different attempt or writer before opening native resources', async () => { + const f = await fixture(); + f.release(); + await f.exited; + await expect( + publishAnalysisAfterProcessExit( + { ...f.handoff, publicationId: 'c'.repeat(32) }, + f.writerPid, + f.actions, + ), + ).rejects.toThrow('attempt changed'); + await expect( + publishAnalysisAfterProcessExit(f.handoff, f.writerPid + 1, f.actions), + ).rejects.toThrow('attempt changed'); + expect(f.actions.verify).not.toHaveBeenCalled(); + }); + + it('preserves the published graph when finalization fails', async () => { + const f = await fixture(); + f.actions.finalizeGraph = async () => { + throw new Error('registry unavailable'); + }; + f.release(); + await expect(f.finish()).rejects.toThrow('registry unavailable'); + const metadata = await loadMeta(f.root); + expect(metadata?.graphGeneration?.id).toBe(f.handoff.publicationId); + const graphFile = metadata?.graphPath; + if (!graphFile) throw new Error('Published graph path is missing'); + expect(await fs.readFile(path.join(f.root, graphFile), 'utf8')).toBe('new graph'); + }); + + it('finishes graph discoverability if cancellation arrives after publication', async () => { + const f = await fixture(); + const controller = new AbortController(); + f.actions.signal = controller.signal; + const finalized = vi.fn(async (metadata: RepoMeta) => { + controller.abort(new Error('cancelled after publication')); + return metadata; + }); + f.actions.finalizeGraph = finalized; + f.release(); + await expect(f.finish()).rejects.toThrow('cancelled after publication'); + expect(finalized).toHaveBeenCalledOnce(); + expect((await loadMeta(f.root))?.graphGeneration?.id).toBe(f.handoff.publicationId); + }); + + it('projects a bounded allowlist without traversing a live graph', async () => { + const f = await fixture(); + const source = { + ...f.handoff, + get pipelineResult(): never { + throw new Error('materialized graph'); + }, + }; + const projected = projectPreparedAnalysisHandoff(source); + expect(projected).toEqual(f.handoff); + expect(JSON.stringify(projected).length).toBeLessThan(2048); + expect(() => + projectPreparedAnalysisHandoff({ ...f.handoff, repoName: 'x'.repeat(1025) }), + ).toThrow('repoName'); + expect(() => + projectPreparedAnalysisHandoff({ ...f.handoff, metadataDirectory: '../foreign' }), + ).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index 2c0323220..e69ec6586 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -36,6 +36,94 @@ const okRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => baseResul const okFinalize: WorkerAnalysisDeps['assertAnalysisFinalized'] = vi.fn(async () => undefined); const alwaysClaim: WorkerAnalysisDeps['claimTerminal'] = () => true; +describe('prepared graph worker handoff', () => { + const handoff = { + version: 1 as const, + publicationId: 'a'.repeat(32), + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + metadataDirectory: '/repo/.gitnexus', + repoName: 'repo', + commit: 'b'.repeat(40), + }; + + it('sends a nonterminal bounded receipt without traversing the graph or finalizing early', async () => { + const claimTerminal = vi.fn(createTerminalClaim()); + const send = vi.fn(); + const finalize = vi.fn(async () => undefined); + const run = vi.fn(async () => ({ + ...baseResult, + preparedGraph: handoff, + get pipelineResult(): never { + throw new Error('live graph traversed'); + }, + })); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: run, + assertAnalysisFinalized: finalize, + claimTerminal, + isCancelled: () => false, + send, + }, + ); + expect(finalize).not.toHaveBeenCalled(); + expect(claimTerminal).not.toHaveBeenCalled(); + expect(claimTerminal()).toBe(true); // A later cancellation still owns one terminal outcome. + expect(send).toHaveBeenCalledOnce(); + const message = send.mock.calls[0][0]; + expect(message.type).toBe('graph-prepared'); + expect(message.handoff).toEqual(handoff); + expect(message.result.preparedGraph).toBeUndefined(); + expect(JSON.stringify(message).length).toBeLessThan(2048); + expect(run.mock.calls[0][1].skipNativeCloseOnExit).toBe(true); + }); + + it('suppresses a prepared receipt after cancellation', async () => { + const send = vi.fn(); + const claimTerminal = vi.fn(); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async () => ({ ...baseResult, preparedGraph: handoff })), + assertAnalysisFinalized: okFinalize, + claimTerminal, + isCancelled: () => true, + send, + }, + ); + expect(send).not.toHaveBeenCalled(); + expect(claimTerminal).not.toHaveBeenCalled(); + }); + + it('refuses a collapsed graph before sending its prepared receipt', async () => { + const send = vi.fn(); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async () => ({ + ...baseResult, + preparedGraph: handoff, + graphWriteCollapsed: { expected: 100, persisted: 1 }, + })), + assertAnalysisFinalized: okFinalize, + claimTerminal: createTerminalClaim(), + isCancelled: () => false, + send, + }, + ); + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith({ + type: 'error', + message: 'Collapsed graph cannot be prepared for publication', + }); + }); +}); + describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { it('reports error (not complete) when finalization fails for an unregistered repo', async () => { const send = vi.fn<(msg: WorkerMessage) => void>(); diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 7c88323cf..32ad10fa7 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -20,6 +20,22 @@ function createChild() { } describe('auto-sync analysis worker', () => { + it('does not treat graph-prepared as terminal or clear the active timeout', async () => { + const child = createChild(); + const clearTimeoutFn = vi.fn(); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn(() => 1) as any, + clearTimeoutFn: clearTimeoutFn as any, + }); + const result = run('/tmp/repo', {}, 50); + child.emit('message', { type: 'graph-prepared', handoff: {}, result: { stats: { files: 3 } } }); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + child.emit('message', { type: 'error', message: 'cancelled after preparing' }); + child.emit('exit', 0, null); + await expect(result).rejects.toThrow('cancelled after preparing'); + }); + it('ignores progress and resolves from the terminal complete message', async () => { const child = createChild(); const forkWorker = vi.fn(() => child as any); From 15930595d690fe813faf8eea440fdd5901f7e58d Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 08:30:08 +0000 Subject: [PATCH 18/67] feat(cli): defer success across graph handoff Finish live-graph skill generation before the child hands off. Transfer only bounded counters, identity and the pre-run self-commit safety flags; wait for IPC delivery before returning for native process exit. Reject missing supervisors and mismatched prepared results before skill writes. Finalization, self-commit and success remain deferred until the production supervisor completes embedding; core activation follows the pending-artifact and resource contracts. Refs #3598 --- gitnexus/src/cli/analyze-phase-ipc.ts | 114 ++++++++ gitnexus/src/cli/analyze.ts | 30 +- .../test/unit/analyze-phase-handoff.test.ts | 275 ++++++++++++++++++ 3 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 gitnexus/src/cli/analyze-phase-ipc.ts create mode 100644 gitnexus/test/unit/analyze-phase-handoff.test.ts diff --git a/gitnexus/src/cli/analyze-phase-ipc.ts b/gitnexus/src/cli/analyze-phase-ipc.ts new file mode 100644 index 000000000..aa7a6c540 --- /dev/null +++ b/gitnexus/src/cli/analyze-phase-ipc.ts @@ -0,0 +1,114 @@ +/** Bounded CLI graph handoff. Live graph consumers run before this is sent. */ +import path from 'node:path'; +import type { AnalyzeResult } from '../core/run-analyze.js'; +import { + projectPreparedAnalysisHandoff, + type PreparedAnalysisHandoff, +} from '../core/analysis-phase-supervisor.js'; +import { FTS_SKIP_REASONS } from '../core/search/fts-policy.js'; + +export interface CliPreparedAnalysisMessage { + type: 'graph-prepared'; + handoff: PreparedAnalysisHandoff; + result: Pick< + AnalyzeResult, + | 'repoPath' + | 'repoName' + | 'storagePath' + | 'stats' + | 'ftsSkipped' + | 'ftsSkipReason' + | 'isPrimaryBranch' + >; + startedAt: number; + /** Snapshot from BEFORE graph/context generation, never recomputed on resume. */ + selfCommitSafety?: { agentsMd: boolean; claudeMd: boolean }; +} + +export function projectCliPreparedAnalysis( + result: AnalyzeResult, + repoPath: string, + startedAt: number, + selfCommitSafety?: ReadonlyMap, +): CliPreparedAnalysisMessage { + const handoff = projectPreparedAnalysisHandoff(result.preparedGraph); + if ( + result.graphWriteCollapsed || + result.alreadyUpToDate || + result.ftsRepairedOnly || + result.repoName !== handoff.repoName || + path.resolve(result.repoPath) !== path.resolve(repoPath) || + path.resolve(result.repoPath) !== path.resolve(handoff.repoPath) || + path.resolve(result.storagePath) !== path.resolve(handoff.storagePath) + ) + throw new Error('Prepared CLI graph does not match the requested analysis'); + if (!Number.isSafeInteger(startedAt) || startedAt < 0) + throw new Error('Invalid graph phase start time'); + + // Never spread stats or the result: either may gain corpus-sized fields. + const stats: AnalyzeResult['stats'] = {}; + for (const key of [ + 'files', + 'nodes', + 'edges', + 'communities', + 'processes', + 'embeddings', + ] as const) { + const value = result.stats[key]; + if (value === undefined) continue; + if (!Number.isSafeInteger(value) || value < 0) + throw new Error(`Invalid graph phase counter: ${key}`); + stats[key] = value; + } + for (const key of ['ftsSkipped', 'isPrimaryBranch'] as const) { + if (result[key] !== undefined && typeof result[key] !== 'boolean') + throw new Error(`Invalid graph phase flag: ${key}`); + } + if (result.ftsSkipReason !== undefined && !FTS_SKIP_REASONS.includes(result.ftsSkipReason)) + throw new Error('Invalid graph phase FTS reason'); + return { + type: 'graph-prepared', + handoff, + result: { + repoPath: handoff.repoPath, + repoName: handoff.repoName, + storagePath: handoff.storagePath, + stats, + ftsSkipped: result.ftsSkipped, + ftsSkipReason: result.ftsSkipReason, + isPrimaryBranch: result.isPrimaryBranch, + }, + startedAt, + selfCommitSafety: selfCommitSafety + ? { + agentsMd: selfCommitSafety.get('AGENTS.md') === true, + claudeMd: selfCommitSafety.get('CLAUDE.md') === true, + } + : undefined, + }; +} + +interface SupervisorChannel { + connected?: boolean; + send?: (message: CliPreparedAnalysisMessage, callback: (error: Error | null) => void) => unknown; +} + +export function requireCliAnalysisSupervisor(channel: SupervisorChannel = process) { + if (typeof channel.send !== 'function' || channel.connected !== true) + throw new Error('Prepared graph requires a connected analysis supervisor'); + return channel.send.bind(channel); +} + +/** A successful send is only delivery, never graph publication or process exit. */ +export async function sendCliPreparedAnalysis( + message: CliPreparedAnalysisMessage, + channel: SupervisorChannel = process, +): Promise { + const send = requireCliAnalysisSupervisor(channel); + await new Promise((resolve, reject) => { + // Await the callback before the caller exits. A false return means channel + // backpressure, not failure; the callback remains the delivery boundary. + send(message, (error: Error | null) => (error ? reject(error) : resolve())); + }); +} diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 07351dee5..24d3b2ad4 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -51,6 +51,7 @@ import { GitNexusRcError, } from './analyze-config.js'; import type { AnalyzeOptions } from './analyze-options.js'; +import type { CliPreparedAnalysisMessage } from './analyze-phase-ipc.js'; import { runFullAnalysis } from '../core/run-analyze.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; @@ -1623,6 +1624,15 @@ const analyzeCommandImpl = async ( : undefined; const result = await runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); + let preparedMessage: CliPreparedAnalysisMessage | undefined; + if (result.preparedGraph) { + const { projectCliPreparedAnalysis, requireCliAnalysisSupervisor } = + await import('./analyze-phase-ipc.js'); + requireCliAnalysisSupervisor(); + // Validate identity before skill/context writes use the registered name. + preparedMessage = projectCliPreparedAnalysis(result, repoPath, t0, selfCommitSafety); + } + if (result.alreadyUpToDate) { // Even the fast path must prove the repo is discoverable. A prior // run can write meta.json and then fail before registerRepo(); in @@ -1701,7 +1711,7 @@ const analyzeCommandImpl = async ( // success so the silent-finalize state surfaces with a non-zero // exit code and an actionable error instead of being mistaken for // a healthy index. - await assertAnalysisFinalized(repoPath, result.storagePath); + if (!preparedMessage) await assertAnalysisFinalized(repoPath, result.storagePath); // Skill generation (CLI-only, uses pipeline result from analysis). // Gated so `--index-only --skills` skips community skill writes too @@ -1765,6 +1775,24 @@ const analyzeCommandImpl = async ( } } + if (preparedMessage) { + // Skill generation above needs the live graph and stays in this child. + // Publication, embeddings, the success banner and self-commit belong to + // the supervisor only after this process has actually exited. + const { sendCliPreparedAnalysis } = await import('./analyze-phase-ipc.js'); + await sendCliPreparedAnalysis(preparedMessage); + clearInterval(elapsedTimer); + process.removeListener('SIGINT', sigintHandler); + console.log = origLog; + // eslint-disable-next-line no-console -- restoring progress-bar routing + console.warn = origWarn; + // eslint-disable-next-line no-console -- restoring progress-bar routing + console.error = origError; + bar.stop(); + process.exitCode = 0; + return; + } + // #2639: opt-in self-commit of any AGENTS.md/CLAUDE.md churn written by // this run (the primary generateAIContextFiles call inside // runFullAnalysis, and/or the --skills regeneration above). Best-effort diff --git a/gitnexus/test/unit/analyze-phase-handoff.test.ts b/gitnexus/test/unit/analyze-phase-handoff.test.ts new file mode 100644 index 000000000..72622e0c6 --- /dev/null +++ b/gitnexus/test/unit/analyze-phase-handoff.test.ts @@ -0,0 +1,275 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { AnalyzeResult } from '../../src/core/run-analyze.js'; +import type { CliPreparedAnalysisMessage } from '../../src/cli/analyze-phase-ipc.js'; + +type SendCallback = (error: Error | null) => void; + +const mocks = vi.hoisted(() => ({ + run: vi.fn(), + skills: vi.fn(async () => ({ + skills: [{ name: 'community' }], + outputPath: '/repo/.claude/skills', + })), + context: vi.fn(async () => ({ files: [] })), + finalized: vi.fn(async () => undefined), + commit: vi.fn(), + snapshot: vi.fn( + () => + new Map([ + ['AGENTS.md', false], + ['CLAUDE.md', true], + ]), + ), + nativeReady: vi.fn(() => false), + channel: { + connected: true, + send: vi.fn((_message: unknown, callback: SendCallback) => callback(null)), + }, +})); +vi.mock('../../src/cli/analyze-phase-ipc.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requireCliAnalysisSupervisor: () => actual.requireCliAnalysisSupervisor(mocks.channel), + sendCliPreparedAnalysis: (message: CliPreparedAnalysisMessage) => + actual.sendCliPreparedAnalysis(message, mocks.channel), + }; +}); +vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: mocks.run })); +vi.mock('../../src/cli/ai-context.js', () => ({ generateAIContextFiles: mocks.context })); +vi.mock('../../src/cli/skill-gen.js', () => ({ generateSkillFiles: mocks.skills })); +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + closeLbug: vi.fn(), + closeLbugBeforeExit: vi.fn(), + isLbugReady: mocks.nativeReady, + LbugWipeError: class LbugWipeError extends Error {}, +})); +vi.mock('../../src/storage/repo-manager.js', () => ({ + getGlobalRegistryPath: vi.fn(() => '/nonexistent/registry.json'), + RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, + AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, + assertAnalysisFinalized: mocks.finalized, +})); +vi.mock('../../src/storage/git.js', () => ({ + getGitRoot: vi.fn(() => '/repo'), + hasGitDir: vi.fn(() => true), + getDefaultBranch: vi.fn(() => null), + selfCommitContextFiles: mocks.commit, + snapshotSelfCommitSafety: mocks.snapshot, +})); +vi.mock('../../src/core/ingestion/utils/max-file-size.js', () => ({ + getMaxFileSizeBannerMessage: vi.fn(() => null), +})); + +const send = mocks.channel.send; +const baselineUnhandled = process.listeners('unhandledRejection'); +const baselineUncaught = process.listeners('uncaughtException'); +let result: AnalyzeResult; + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv('NODE_OPTIONS', '--max-old-space-size=8192'); + process.exitCode = undefined; + mocks.nativeReady.mockReturnValue(false); + result = { + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + repoName: 'registered-repo', + stats: { files: 1, nodes: 10, edges: 20, communities: 1, processes: 2 }, + rebuildReasons: [], + preparedGraph: { + version: 1, + publicationId: 'a'.repeat(32), + commit: 'b'.repeat(40), + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + metadataDirectory: '/repo/.gitnexus', + repoName: 'registered-repo', + }, + pipelineResult: { + graph: { + toJSON: () => { + throw new Error('Live graph serialized'); + }, + }, + }, + }; + mocks.run.mockResolvedValue(result); + send.mockImplementation((message, callback) => { + JSON.stringify(message); + callback(null); + }); + mocks.channel.connected = true; + vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + vi.spyOn(console, 'log').mockImplementation(() => undefined); +}); + +afterEach(() => { + for (const listener of process.listeners('unhandledRejection')) { + if (!baselineUnhandled.includes(listener)) + process.removeListener('unhandledRejection', listener); + } + for (const listener of process.listeners('uncaughtException')) { + if (!baselineUncaught.includes(listener)) process.removeListener('uncaughtException', listener); + } + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + process.exitCode = undefined; +}); + +describe('CLI graph phase handoff', () => { + it('finishes live-graph skills before IPC and defers finalization, self-commit and success', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const output = console.log; + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + + expect(send).toHaveBeenCalledTimes(1); + expect(mocks.snapshot.mock.invocationCallOrder[0]).toBeLessThan( + mocks.run.mock.invocationCallOrder[0], + ); + expect(mocks.skills).toHaveBeenCalledWith('/repo', 'registered-repo', result.pipelineResult); + expect(mocks.context.mock.invocationCallOrder[0]).toBeLessThan( + send.mock.invocationCallOrder[0], + ); + expect(send.mock.calls[0][0]).toMatchObject({ + type: 'graph-prepared', + handoff: result.preparedGraph, + selfCommitSafety: { agentsMd: false, claudeMd: true }, + result: { repoName: 'registered-repo', stats: result.stats }, + }); + expect(JSON.stringify(send.mock.calls[0][0]).length).toBeLessThan(2048); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(output).not.toHaveBeenCalledWith(expect.stringContaining('indexed successfully')); + expect(process.exitCode).toBe(0); + }); + + it('keeps --index-only skill suppression and does not snapshot self-commit when omitted', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, indexOnly: true }); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.context).not.toHaveBeenCalled(); + expect(mocks.snapshot).not.toHaveBeenCalled(); + expect(send.mock.calls[0][0]).not.toHaveProperty('selfCommitSafety', expect.anything()); + expect(process.exitCode).toBe(0); + }); + + it('waits for IPC delivery before the native-handle exit, including channel backpressure', async () => { + let delivered!: SendCallback; + let received!: () => void; + const receiving = new Promise((resolve) => { + received = resolve; + }); + send.mockImplementationOnce((_message, callback) => { + delivered = callback; + received(); + return false; + }); + mocks.nativeReady.mockReturnValue(true); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const running = analyzeCommand(undefined, {}); + await receiving; + expect(process.exit).not.toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + delivered(null); + await running; + expect(process.exit).toHaveBeenCalledExactlyOnceWith(0); + }); + + it.each(['throw', 'callback'] as const)( + 'reports an IPC %s failure without success or self-commit', + async (mode) => { + send.mockImplementationOnce((_message, callback) => { + const error = new Error('Supervisor disconnected'); + if (mode === 'throw') throw error; + callback(error); + }); + mocks.nativeReady.mockReturnValue(true); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const output = console.log; + await analyzeCommand(undefined, { selfCommit: true }); + expect(process.exit).toHaveBeenCalledExactlyOnceWith(1); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(output).not.toHaveBeenCalledWith(expect.stringContaining('indexed successfully')); + }, + ); + + it('fails before skill writes when the supervisor is missing', async () => { + mocks.channel.connected = false; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('fails if the supervisor disconnects during child-local postprocessing', async () => { + mocks.context.mockImplementationOnce(async () => { + mocks.channel.connected = false; + return { files: [] }; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).toHaveBeenCalledOnce(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it.each([ + { repoName: 'different-registration' }, + { repoPath: '/other' }, + { storagePath: '/other' }, + { alreadyUpToDate: true }, + { ftsRepairedOnly: true }, + { graphWriteCollapsed: { expected: 20, persisted: 0 } }, + ])('rejects an incompatible result before skills: %j', async (invalid) => { + Object.assign(result, invalid); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('projects only named counters and safety flags, without traversing future payload fields', async () => { + const poison = { + enumerable: true, + get() { + throw new Error('Unbounded field traversed'); + }, + }; + Object.defineProperty(result.stats, 'perFile', poison); + Object.defineProperty(result.preparedGraph, 'sourceMaps', poison); + Object.defineProperty(result, 'sourceMaps', poison); + mocks.snapshot.mockReturnValueOnce( + new Map([ + ['unrelated-file', true], + ['CLAUDE.md', true], + ]), + ); + result.ftsSkipped = true; + result.ftsSkipReason = 'disabled-by-flag'; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { selfCommit: true }); + expect(send.mock.calls[0][0]).toMatchObject({ + selfCommitSafety: { agentsMd: false, claudeMd: true }, + result: { ftsSkipped: true, ftsSkipReason: 'disabled-by-flag' }, + }); + expect(JSON.stringify(send.mock.calls[0][0])).not.toMatch(/perFile|sourceMaps|unrelated-file/); + expect(process.exitCode).toBe(0); + }); + + it.each([NaN, Infinity, -1, 1.5])('rejects invalid summary counter %s', async (nodes) => { + result.stats.nodes = nodes; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); +}); From 3ea7e37697dd1b3e2f5ad5a9016a079505a9804f Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 09:28:28 +0000 Subject: [PATCH 19/67] feat(embeddings): rebase pending paid artifacts Separate graph job identity from mutable artifact ownership. Keep paid compatible groups across source edits using bounded revalidation and selection pruning; retain one sealed fallback for the active space. --- .../core/embeddings/embedding-job-runner.ts | 33 ++- gitnexus/src/core/embeddings/embedding-job.ts | 211 ++++++++++++--- .../embeddings/vector-generation-search.ts | 5 +- .../src/core/embeddings/vector-publication.ts | 9 +- gitnexus/src/core/lbug/vector-store.ts | 116 ++++++++ gitnexus/src/storage/embedding-artifact.ts | 249 ++++++++++++++++++ gitnexus/src/storage/embedding-job.ts | 2 +- gitnexus/src/storage/repo-meta.ts | 11 +- gitnexus/src/storage/vector-generation.ts | 9 +- .../embedding-job-controller.test.ts | 235 +++++++++++++++++ .../embedding-vector-store.test.ts | 116 ++++++++ gitnexus/test/unit/embedding-artifact.test.ts | 206 +++++++++++++++ gitnexus/test/unit/vector-generation.test.ts | 26 ++ 13 files changed, 1182 insertions(+), 46 deletions(-) create mode 100644 gitnexus/src/storage/embedding-artifact.ts create mode 100644 gitnexus/test/unit/embedding-artifact.test.ts diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index c8018e591..c6f949f91 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -1,6 +1,6 @@ /** One durable manifest unit, one bounded provider batch, no corpus-sized maps. */ import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; -import type { VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; +import type { VectorNodeReceipt, VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; import type { EmbeddingJobJournal, EmbeddingJobState, @@ -29,6 +29,9 @@ export interface EmbeddingJobRunnerOptions { store: VectorStore; /** The controller has validated vector/input identities before providing reuse. */ reuse?: Pick; + /** The writer was adopted from an older graph. Persist this choice for the + * whole job: old complete receipts cannot skip current-input validation. */ + revalidateStoredInputs?: boolean; config?: Partial; queueBytes: number; provider: typeof embedBatch; @@ -78,7 +81,9 @@ export async function runEmbeddingJob( intent: EmbeddingUnitIntent, ): Promise => { signal?.throwIfAborted(); - const complete = await store.completedNodes(unit.nodes.map((node) => node.id)); + const complete = options.revalidateStoredInputs + ? new Map() + : await store.completedNodes(unit.nodes.map((node) => node.id)); const missing = unit.nodes.filter((node) => !complete.has(node.id)); const retainedBytes = missing.reduce( (n, key) => n + 2 * (key.contentBytes + key.metadataBytes), @@ -204,11 +209,24 @@ export async function runEmbeddingJob( sourceStartLine: node.startLine ?? 0, sourceEndLine: node.endLine ?? 0, })); - await store.beginNodes(inputs); + if (options.revalidateStoredInputs) { + for (const input of inputs) { + const retained = await store.reuseNode(input); + if (retained) { + complete.set(input.nodeId, retained); + reusedNodes++; + } + } + } + const missingInputs = options.revalidateStoredInputs + ? inputs.filter((input) => !complete.has(input.nodeId)) + : inputs; + if (missingInputs.length) await store.beginNodes(missingInputs); for (let i = 0; i < fetched.nodes.length; i++) { const node = fetched.nodes[i]; const input = inputs[i]; signal?.throwIfAborted(); + if (options.revalidateStoredInputs && complete.has(node.id)) continue; const reuse = options.reuse; const reusable = await reuse?.reusable(node.id, input.contentHash); if (reusable && reuse) { @@ -329,7 +347,11 @@ export async function runEmbeddingJob( if (finished.length) for (const receipt of await store.finishNodes(finished)) complete.set(receipt.nodeId, receipt); - if (complete.size) await store.checkpointUnit([...complete.values()]); + if (complete.size) { + if (options.revalidateStoredInputs) + await store.markSelected(journal.snapshot().identity, [...complete.keys()]); + await store.checkpointUnit([...complete.values()]); + } if (complete.size === unit.nodes.length) { await journal.acknowledge({ reusedNodes, @@ -381,6 +403,9 @@ export async function runEmbeddingJob( progress(); } signal?.throwIfAborted(); + if (options.revalidateStoredInputs && !store.isSealed) + await store.pruneUnselected(journal.snapshot().identity, manifest.totalNodes); + signal?.throwIfAborted(); const indexKind = await store.seal(manifest.totalNodes, journal.snapshot().chunks); return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; } diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 4ce17cc4a..196dcbd77 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -1,9 +1,15 @@ /** Common lifecycle for jobs on a published graph. The caller holds its index lock. */ import fs from 'node:fs/promises'; import path from 'node:path'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import type { RepoMeta } from '../../storage/repo-meta.js'; import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; +import { + adoptEmbeddingArtifact, + embeddingArtifactSpaceKey, + readEmbeddingArtifactSpace, + type EmbeddingArtifactOwner, +} from '../../storage/embedding-artifact.js'; import { EmbeddingJobJournal, type EmbeddingJobState, @@ -192,7 +198,7 @@ export async function runPublishedEmbeddingJob( ); const previous = metadata.vectorGeneration; if (previous?.identity === identities.identity && !metadata.embeddingJob) { - await assertOwnedVectorGeneration(root, previous.identity); + await assertOwnedVectorGeneration(root, previous.artifactIdentity ?? previous.identity); return { metadata, alreadyComplete: true, phase: 'ready', peakQueueBytes: 0 }; } if (metadata.embeddingJob && metadata.embeddingJob.identity !== identities.identity) { @@ -208,34 +214,57 @@ export async function runPublishedEmbeddingJob( } else if (previous && previous.inputIdentity !== identities.inputIdentity) { options.log?.('Embedding text options changed; prior vectors are not compatible.'); } - const reusable = + const previousCompatible = previous && previous.identity !== identities.identity && previous.vectorIdentity === identities.vectorIdentity && previous.inputIdentity === identities.inputIdentity && previous.dimensions === vector.dimensions; - // A reuse reader shares the vector reservation; it is not a third default pool. - const poolBytes = Math.floor(resources.vectorPoolBytes / (reusable ? 2 : 1)); - const storeResources = { - bufferPoolBytes: poolBytes, - threads: resources.nativeThreads, - batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), - }; - if (poolBytes < 8 * 1024 * 1024 || storeResources.batchBytes < vector.dimensions * 16) - throw new Error('Vector writer and reuse reader do not fit their resource reservation'); - const directory = await jobDirectory(root, identities.identity); - if (metadata.embeddingJob?.identity !== identities.identity) { - metadata = { - ...metadata, - embeddingJob: { - version: 1, - identity: identities.identity, - graphGeneration: graph.id, - }, + const limits = (hasFallback: boolean) => { + // Writer and the sole reuse reader share this reservation. A sealed probe + // is physically closed before its replacement writer is opened. + const result = { + bufferPoolBytes: Math.floor(resources.vectorPoolBytes / (hasFallback ? 2 : 1)), + threads: resources.nativeThreads, + batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), }; - await options.metadata.saveMetadata(metadata); - await syncPublishedMetadata(root, metadata); - } + if (result.bufferPoolBytes < 8 * 1024 * 1024 || result.batchBytes < vector.dimensions * 16) + throw new Error('Vector writer and reuse reader do not fit their resource reservation'); + return result; + }; + const space = { + vectorIdentity: identities.vectorIdentity, + inputIdentity: identities.inputIdentity, + dimensions: vector.dimensions, + schema: VECTOR_STORE_SCHEMA, + }; + const validateOwner = async (owner: EmbeddingArtifactOwner) => { + const directory = await embeddingJobDirectory(path.join(root, 'embeddings', owner.jobIdentity)); + const manifest = await loadEmbeddingWorkManifest(directory, owner.sourceIdentity); + if (manifest.digest !== owner.manifestDigest) + throw new Error('Owned embedding artifact manifest changed; paid artifacts retained'); + }; + let artifact = await readEmbeddingArtifactSpace(root, space, validateOwner); + if ( + !artifact && + metadata.embeddingJob && + !metadata.embeddingJob.spaceKey && + metadata.embeddingJob.identity !== identities.identity + ) + options.log?.( + 'Prior pending job has no verified vector/input ownership receipt; retaining its files without assuming compatibility.', + ); + const needsFallback = () => + artifact + ? artifact.owner.phase === 'sealed' + ? artifact.owner.jobIdentity !== identities.identity + : Boolean(artifact.owner.fallbackArtifact) + : Boolean( + previousCompatible && + (previous.artifactIdentity ?? previous.identity) !== identities.identity, + ); + limits(needsFallback()); + const directory = await jobDirectory(root, identities.identity); let source: GraphSource | undefined; let store: VectorStore | undefined; let reuse: VectorStore | undefined; @@ -248,7 +277,44 @@ export async function runPublishedEmbeddingJob( const { drainEmbeddingSidecarForAnalysis } = await import('./embedding-sidecar-client.js'); await drainEmbeddingSidecarForAnalysis(); } + // A crash after native sealing but before recording that fact must not + // turn the complete artifact into a mutable writer for a different graph. + if (artifact?.owner.phase === 'pending' && artifact.owner.jobIdentity !== identities.identity) { + const ownedDirectory = await jobDirectory(root, artifact.owner.artifactIdentity); + if (await existsOwnedFile(path.join(ownedDirectory, 'vectors'))) { + store = await VectorStore.open({ + directory: ownedDirectory, + identity: artifact.owner.artifactIdentity, + dimensions: vector.dimensions, + mode: 'resume', + recoverEmptyBootstrap: !(await existsOwnedFile( + path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json'), + )), + ...limits(Boolean(artifact.owner.fallbackArtifact)), + }); + if (store.isSealed) { + await store.close(); + store = undefined; + artifact = await adoptEmbeddingArtifact( + root, + space, + artifact, + { ...artifact.owner, phase: 'sealed', fallbackArtifact: undefined }, + validateOwner, + ); + } + } else if ( + await existsOwnedFile(path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json')) + ) { + throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); + } + } + // Sealing may have been discovered by the probe. Admit both pools before + // reading source or creating a replacement manifest. + limits(needsFallback()); const manifestExists = await existsOwnedFile(path.join(directory, 'manifest.json')); + if (!manifestExists && artifact?.owner.jobIdentity === identities.identity) + throw new Error('Owned embedding manifest is missing; paid artifacts retained'); if (!manifestExists) await clearUnstartedManifest(directory); const getSource = async (): Promise => (source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)( @@ -259,24 +325,91 @@ export async function runPublishedEmbeddingJob( const manifest = manifestExists ? await loadEmbeddingWorkManifest(directory, identities.sourceIdentity) : await createEmbeddingWorkManifest(directory, identities.sourceIdentity, await getSource()); - const hasStore = await existsOwnedFile(path.join(directory, 'vectors')); + const makeOwner = ( + artifactIdentity: string, + revalidateInputs: boolean, + fallbackArtifact?: string, + ): EmbeddingArtifactOwner => ({ + artifactIdentity, + jobIdentity: identities.identity, + graphGeneration: graph.id, + sourceIdentity: identities.sourceIdentity, + manifestDigest: manifest.digest, + phase: 'pending', + revalidateInputs, + ...(fallbackArtifact === undefined ? {} : { fallbackArtifact }), + }); + let owner: EmbeddingArtifactOwner; + if (artifact?.owner.phase === 'sealed' && artifact.owner.jobIdentity === identities.identity) { + // Publication/index-only recovery for this exact manifest. Never reopen + // a sealed artifact with a native writer handle. + owner = artifact.owner; + } else if (artifact?.owner.phase === 'sealed') { + owner = makeOwner(randomBytes(32).toString('hex'), false, artifact.owner.artifactIdentity); + } else if (artifact) { + owner = makeOwner( + artifact.owner.artifactIdentity, + artifact.owner.revalidateInputs || artifact.owner.jobIdentity !== identities.identity, + artifact.owner.fallbackArtifact, + ); + } else { + // Start a space beside its first job. This also adopts an older bounded + // artifact only for this exact compatible job; a digest cannot tell us + // the vector/text provenance of a different old pending job. + const legacyStore = await existsOwnedFile(path.join(directory, 'vectors')); + owner = makeOwner( + identities.identity, + false, + previousCompatible ? (previous.artifactIdentity ?? previous.identity) : undefined, + ); + if (owner.fallbackArtifact === owner.artifactIdentity) owner.fallbackArtifact = undefined; + if ( + legacyStore && + previous?.identity === identities.identity && + (previous.artifactIdentity ?? previous.identity) === owner.artifactIdentity + ) + owner.phase = 'sealed'; + } + artifact = await adoptEmbeddingArtifact(root, space, artifact, owner, validateOwner); + const storeResources = limits(Boolean(owner.fallbackArtifact)); + if ( + metadata.embeddingJob?.identity !== identities.identity || + metadata.embeddingJob.artifactIdentity !== owner.artifactIdentity || + metadata.embeddingJob.spaceKey !== artifact.spaceKey + ) { + metadata = { + ...metadata, + embeddingJob: { + version: 1, + identity: identities.identity, + graphGeneration: graph.id, + artifactIdentity: owner.artifactIdentity, + spaceKey: embeddingArtifactSpaceKey(space), + }, + }; + await options.metadata.saveMetadata(metadata); + await syncPublishedMetadata(root, metadata); + } + signal?.throwIfAborted(); + const artifactDirectory = await jobDirectory(root, owner.artifactIdentity); + const hasStore = await existsOwnedFile(path.join(artifactDirectory, 'vectors')); const hasJournal = await existsOwnedFile(path.join(directory, 'job.json')); if (!hasStore && hasJournal) throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); - store = await VectorStore.open({ - directory, - identity: identities.identity, + store ??= await VectorStore.open({ + directory: artifactDirectory, + identity: owner.artifactIdentity, dimensions: vector.dimensions, - mode: hasStore ? 'resume' : 'create', + mode: owner.phase === 'sealed' ? 'read' : hasStore ? 'resume' : 'create', recoverEmptyBootstrap: !hasJournal, ...storeResources, }); const journal = await EmbeddingJobJournal.recover(directory, identities.identity, manifest); - if (reusable) { - const file = await assertOwnedVectorGeneration(root, previous.identity); + if (owner.fallbackArtifact) { + const file = await assertOwnedVectorGeneration(root, owner.fallbackArtifact); reuse = await VectorStore.open({ directory: path.dirname(file), - identity: previous.identity, + identity: owner.fallbackArtifact, dimensions: vector.dimensions, mode: 'read', ...storeResources, @@ -302,6 +435,9 @@ export async function runPublishedEmbeddingJob( journal, store, reuse, + // A sealed artifact still belongs to this exact job/manifest; rebuilding + // a damaged cursor can use its complete groups without source reads. + revalidateStoredInputs: owner.revalidateInputs && !store.isSealed, config, queueBytes: resources.queueBytes, // Opening the graph is lazy even on resume: indexing-only retries do not @@ -317,6 +453,7 @@ export async function runPublishedEmbeddingJob( signal, onProgress: options.onProgress, }); + const sealed = store.isSealed; await store.close(); store = undefined; await reuse?.close(); @@ -326,11 +463,21 @@ export async function runPublishedEmbeddingJob( signal?.throwIfAborted(); if (!result.indexKind) return { metadata, alreadyComplete: false, phase: 'embedding', ...result }; + if (sealed && artifact.owner.phase !== 'sealed') { + artifact = await adoptEmbeddingArtifact( + root, + space, + artifact, + { ...artifact.owner, phase: 'sealed', fallbackArtifact: undefined }, + validateOwner, + ); + } const published = await publishVectorGeneration( root, { version: 1, identity: identities.identity, + artifactIdentity: owner.artifactIdentity, graphGeneration: graph.id, vectorIdentity: identities.vectorIdentity, inputIdentity: identities.inputIdentity, diff --git a/gitnexus/src/core/embeddings/vector-generation-search.ts b/gitnexus/src/core/embeddings/vector-generation-search.ts index 92323616d..dc09f7464 100644 --- a/gitnexus/src/core/embeddings/vector-generation-search.ts +++ b/gitnexus/src/core/embeddings/vector-generation-search.ts @@ -49,9 +49,10 @@ export async function searchVectorGeneration( if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000 || !Number.isFinite(maxDistance)) throw new Error('Invalid vector search bounds'); const dbPath = selected.path; + const artifactIdentity = selected.generation.artifactIdentity ?? selected.generation.identity; const release = pinRepo(dbPath); try { - await assertOwnedVectorGeneration(snapshot.metadataDirectory, selected.generation.identity); + await assertOwnedVectorGeneration(snapshot.metadataDirectory, artifactIdentity); await initLbug(dbPath, dbPath, { kind: 'vector', bufferPoolBytes: 64 * 1024 * 1024, @@ -65,7 +66,7 @@ export async function searchVectorGeneration( const proof = artifact[0]; if ( artifact.length !== 1 || - proof.identity !== selected.generation.identity || + proof.identity !== artifactIdentity || proof.schema !== VECTOR_STORE_SCHEMA || Number(proof.dimensions) !== selected.generation.dimensions || proof.complete !== true || diff --git a/gitnexus/src/core/embeddings/vector-publication.ts b/gitnexus/src/core/embeddings/vector-publication.ts index c0a514162..ab2e53e96 100644 --- a/gitnexus/src/core/embeddings/vector-publication.ts +++ b/gitnexus/src/core/embeddings/vector-publication.ts @@ -32,7 +32,9 @@ export async function publishVectorGeneration( previous.incrementalInProgress || previous.graphGeneration.id !== generation.graphGeneration || previous.embeddingJob?.identity !== generation.identity || - previous.embeddingJob?.graphGeneration !== generation.graphGeneration + previous.embeddingJob?.graphGeneration !== generation.graphGeneration || + (previous.embeddingJob.artifactIdentity ?? previous.embeddingJob.identity) !== + (generation.artifactIdentity ?? generation.identity) ) throw new Error('Vector publication does not match the active graph job'); const metadata: RepoMeta = { @@ -67,10 +69,11 @@ export async function publishVectorGeneration( !(selection.kind === 'unavailable' && selection.reason === 'embeddings-empty') ) throw new Error('Invalid vector publication receipt'); - const file = await assertOwnedVectorGeneration(metadataDirectory, generation.identity); + const artifactIdentity = generation.artifactIdentity ?? generation.identity; + const file = await assertOwnedVectorGeneration(metadataDirectory, artifactIdentity); const store = await VectorStore.open({ directory: path.dirname(file), - identity: generation.identity, + identity: artifactIdentity, dimensions: generation.dimensions, mode: 'read', ...resources, diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index d35390e58..37afe7489 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -69,6 +69,11 @@ export class VectorStore { private closing?: Promise; private poisoned = false; private sealed = false; + private selectionReady = false; + + get isSealed(): boolean { + return this.sealed; + } private constructor( private readonly handle: LbugConnectionHandle, @@ -249,6 +254,117 @@ export class VectorStore { return receipts; } + /** Reuse a complete group in the pending writer after validating its CURRENT + * input. Relocation updates only line metadata, with no vector round trip. + * One transaction makes an interrupted relocation safe to retry. */ + async reuseNode(input: VectorNodeInput): Promise { + assertValidGraphNodeId(input.nodeId, 'Vector store'); + const startLine = input.sourceStartLine ?? 0; + const endLine = input.sourceEndLine ?? 0; + if ( + !input.contentHash || + !nonnegative(startLine) || + !nonnegative(endLine) || + endLine < startLine + ) + throw new Error('Invalid reusable embedding input'); + return this.exclusive(async () => { + const previous = await this.receipt(input.nodeId); + if (!previous?.complete || previous.chunks < 1 || previous.contentHash !== input.contentHash) + return null; + await this.verify(previous); + if (previous.sourceStartLine === startLine && previous.sourceEndLine === endLine) + return previous; + const delta = startLine - (previous.sourceStartLine ?? 0); + await this.transaction(async () => { + for (let start = 0; start < previous.chunks; start += BLOCK_ROWS) { + const ids = Array.from( + { length: Math.min(BLOCK_ROWS, previous.chunks - start) }, + (_, i) => `${input.nodeId}:${start + i}`, + ); + await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.startLine=e.startLine+$delta, e.endLine=CASE WHEN e.endLine=$oldEnd THEN $endLine ELSE e.endLine+$delta END', + { ids, delta, oldEnd: previous.sourceEndLine ?? 0, endLine }, + ); + const invalid = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) WHERE e.startLine < 0 OR e.endLine < e.startLine RETURN count(e) AS invalid', + { ids }, + ); + if (count(invalid[0].invalid)) throw new Error('Invalid relocated vector lines'); + } + await this.query( + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.sourceStartLine=$startLine, n.sourceEndLine=$endLine', + { nodeId: input.nodeId, startLine, endLine }, + ); + }); + return { ...previous, sourceStartLine: startLine, sourceEndLine: endLine }; + }, true); + } + + private async ensureSelectionTable(): Promise { + if (this.selectionReady) return; + // Auxiliary scheduling data leaves the vector/input schema unchanged. + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS EmbeddingSelection (nodeId STRING, job STRING, PRIMARY KEY(nodeId))', + ); + this.selectionReady = true; + } + + /** Mark only verified complete groups, before the unit checkpoint and cursor. + * Native selection rows replace an unbounded JS set of current node IDs. */ + async markSelected(job: string, ids: readonly string[]): Promise { + if ( + !/^[a-f0-9]{64}$/.test(job) || + !ids.length || + ids.length > 64 || + new Set(ids).size !== ids.length + ) + throw new Error('Invalid vector selection unit'); + for (const id of ids) assertValidGraphNodeId(id, 'Vector selection'); + await this.exclusive(async () => { + const receipts = await this.receipts(ids); + if ( + receipts.size !== ids.length || + [...receipts.values()].some((r) => !r.complete || r.chunks < 1) + ) + throw new Error('Cannot select an incomplete vector group'); + await this.ensureSelectionTable(); + await this.transaction(async () => { + await this.query( + 'UNWIND $ids AS id MERGE (s:EmbeddingSelection {nodeId:id}) SET s.job=$job', + { ids: [...ids], job }, + ); + }); + }, true); + } + + /** Called only after the new manifest is fully acknowledged. Counts must + * match before deletion, and native joins keep pruning independent of JS RAM. */ + async pruneUnselected(job: string, totalNodes: number): Promise { + if (!/^[a-f0-9]{64}$/.test(job) || !nonnegative(totalNodes)) + throw new Error('Invalid vector selection coverage'); + await this.exclusive(async () => { + await this.ensureSelectionTable(); + const selected = await this.query( + 'MATCH (s:EmbeddingSelection) WHERE s.job=$job RETURN count(s) AS nodes', + { job }, + ); + if (count(selected[0].nodes) !== totalNodes) + throw new Error('Vector selection coverage is incomplete; no groups pruned'); + await this.transaction(async () => { + await this.query( + 'MATCH (e:CodeEmbedding) WHERE NOT EXISTS { MATCH (s:EmbeddingSelection) WHERE s.nodeId=e.nodeId AND s.job=$job } DELETE e', + { job }, + ); + await this.query( + 'MATCH (n:EmbeddingNodeReceipt) WHERE NOT EXISTS { MATCH (s:EmbeddingSelection) WHERE s.nodeId=n.nodeId AND s.job=$job } DELETE n', + { job }, + ); + await this.query('MATCH (s:EmbeddingSelection) WHERE s.job<>$job DELETE s', { job }); + }); + }, true); + } + /** Replay replaces the entire old group, using PKs rather than repeated table scans. */ async beginNode(nodeId: string, contentHash: string): Promise { return this.beginNodes([{ nodeId, contentHash }]); diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts new file mode 100644 index 000000000..2c6979a44 --- /dev/null +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -0,0 +1,249 @@ +/** Directly keyed ownership of one pending artifact per compatible input space. + * Caller holds the ordinary index lock. No model, native pool or history scan. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, + syncEmbeddingJobDirectory, +} from './embedding-job.js'; + +export interface EmbeddingArtifactSpace { + vectorIdentity: string; + inputIdentity: string; + dimensions: number; + schema: string; +} + +export interface EmbeddingArtifactOwner { + artifactIdentity: string; + jobIdentity: string; + graphGeneration: string; + sourceIdentity: string; + manifestDigest: string; + phase: 'pending' | 'sealed'; + revalidateInputs: boolean; + /** At most one completed, read-only source for bounded group reuse. */ + fallbackArtifact?: string; +} + +export interface EmbeddingArtifactRecord { + version: 1; + spaceKey: string; + space: EmbeddingArtifactSpace; + owner: EmbeddingArtifactOwner; +} + +type ValidateOwner = (owner: EmbeddingArtifactOwner) => Promise; +const DIGEST = /^[a-f0-9]{64}$/; + +export function embeddingArtifactSpaceKey(space: EmbeddingArtifactSpace): string { + if ( + !DIGEST.test(space.vectorIdentity) || + !DIGEST.test(space.inputIdentity) || + !Number.isSafeInteger(space.dimensions) || + space.dimensions < 1 || + typeof space.schema !== 'string' || + !space.schema.length || + space.schema.length > 256 + ) + throw new Error('Invalid embedding artifact space'); + return createHash('sha256') + .update( + JSON.stringify([ + 1, + space.vectorIdentity, + space.inputIdentity, + space.dimensions, + space.schema, + ]), + ) + .digest('hex'); +} + +function ownerRecord(value: unknown): EmbeddingArtifactOwner { + const owner = value as EmbeddingArtifactOwner; + if ( + !owner || + typeof owner !== 'object' || + !DIGEST.test(owner.artifactIdentity) || + !DIGEST.test(owner.jobIdentity) || + !DIGEST.test(owner.sourceIdentity) || + !DIGEST.test(owner.manifestDigest) || + !/^[a-f0-9]{32}$/.test(owner.graphGeneration) || + !['pending', 'sealed'].includes(owner.phase) || + typeof owner.revalidateInputs !== 'boolean' || + (owner.fallbackArtifact !== undefined && + (!DIGEST.test(owner.fallbackArtifact) || owner.fallbackArtifact === owner.artifactIdentity)) + ) + throw new Error('Invalid embedding artifact ownership; artifacts retained'); + return { + artifactIdentity: owner.artifactIdentity, + jobIdentity: owner.jobIdentity, + graphGeneration: owner.graphGeneration, + sourceIdentity: owner.sourceIdentity, + manifestDigest: owner.manifestDigest, + phase: owner.phase, + revalidateInputs: owner.revalidateInputs, + ...(owner.fallbackArtifact === undefined ? {} : { fallbackArtifact: owner.fallbackArtifact }), + }; +} + +function record(value: unknown, space: EmbeddingArtifactSpace): EmbeddingArtifactRecord { + const input = value as EmbeddingArtifactRecord; + const spaceKey = embeddingArtifactSpaceKey(space); + if ( + !input || + input.version !== 1 || + input.spaceKey !== spaceKey || + !input.space || + embeddingArtifactSpaceKey(input.space) !== spaceKey + ) + throw new Error('Embedding artifact space mismatch; artifacts retained'); + return { + version: 1, + spaceKey, + space: { + vectorIdentity: space.vectorIdentity, + inputIdentity: space.inputIdentity, + dimensions: space.dimensions, + schema: space.schema, + }, + owner: ownerRecord(input.owner), + }; +} + +async function spaceDirectory(root: string, space: EmbeddingArtifactSpace): Promise { + let directory = await embeddingJobDirectory(root); + for (const component of ['embeddings', 'spaces', embeddingArtifactSpaceKey(space)]) { + const next = path.join(directory, component); + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + +async function optionalRecord(directory: string, name: string): Promise { + try { + return await readEmbeddingJobRecord(directory, name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +const same = (a: EmbeddingArtifactRecord | null, b: EmbeddingArtifactRecord | null): boolean => + JSON.stringify(a) === JSON.stringify(b); + +function validateTransition( + before: EmbeddingArtifactRecord | null, + after: EmbeddingArtifactRecord, +): void { + if (!before) return; + const a = before.owner, + b = after.owner; + if (a.artifactIdentity !== b.artifactIdentity) { + if ( + a.phase !== 'sealed' || + b.phase !== 'pending' || + b.fallbackArtifact !== a.artifactIdentity || + b.artifactIdentity === a.fallbackArtifact + ) + throw new Error('Pending paid artifacts must be retained until sealed'); + return; + } + if (a.jobIdentity !== b.jobIdentity) { + if ( + a.phase !== 'pending' || + b.phase !== 'pending' || + !b.revalidateInputs || + a.fallbackArtifact !== b.fallbackArtifact + ) + throw new Error( + 'Rebased artifact requires current-input validation and its retained fallback', + ); + return; + } + if ( + a.graphGeneration !== b.graphGeneration || + a.sourceIdentity !== b.sourceIdentity || + a.manifestDigest !== b.manifestDigest || + (a.phase === 'sealed' && b.phase !== 'sealed') || + (a.revalidateInputs && !b.revalidateInputs) + ) + throw new Error('Embedding artifact job identity or sealed state changed'); +} + +async function finishIntent(directory: string): Promise { + await fs.unlink(path.join(directory, 'rebase.json')); + await syncEmbeddingJobDirectory(directory); +} + +/** A lagging metadata job pointer is reconciled by the controller BEFORE it + * opens this artifact for paid work. The owned receipt names the sole writer. + * Recovery completes only the exact before/after transition, never guessing + * from an old journal cursor or choosing an artifact by directory enumeration. */ +export async function readEmbeddingArtifactSpace( + root: string, + space: EmbeddingArtifactSpace, + validateOwner: ValidateOwner, +): Promise { + const directory = await spaceDirectory(root, space); + const currentRaw = await optionalRecord(directory, 'space.json'); + let current = currentRaw === null ? null : record(currentRaw, space); + const intent = (await optionalRecord(directory, 'rebase.json')) as { + version?: unknown; + before?: unknown; + after?: unknown; + } | null; + if (intent !== null) { + if (intent.version !== 1 || intent.before === undefined || !intent.after) + throw new Error('Invalid embedding rebase intent; artifacts retained'); + const before = intent.before === null ? null : record(intent.before, space); + const after = record(intent.after, space); + validateTransition(before, after); + if (!same(current, before) && !same(current, after)) + throw new Error('Embedding artifact owner superseded its rebase intent; artifacts retained'); + await validateOwner(after.owner); + if (!same(current, after)) await writeEmbeddingJobRecord(directory, 'space.json', after); + await finishIntent(directory); + current = after; + } + if (current) await validateOwner(current.owner); + return current; +} + +/** Manifest creation precedes this durable intent; journal recovery and native + * mutation follow it. A crash at either rename can be replayed by read above. */ +export async function adoptEmbeddingArtifact( + root: string, + space: EmbeddingArtifactSpace, + previous: EmbeddingArtifactRecord | null, + owner: EmbeddingArtifactOwner, + validateOwner: ValidateOwner, +): Promise { + const directory = await spaceDirectory(root, space); + const before = previous === null ? null : record(previous, space); + const after = record( + { version: 1, spaceKey: embeddingArtifactSpaceKey(space), space, owner }, + space, + ); + const currentRaw = await optionalRecord(directory, 'space.json'); + const current = currentRaw === null ? null : record(currentRaw, space); + if (!same(current, before) || (await optionalRecord(directory, 'rebase.json')) !== null) + throw new Error('Embedding artifact ownership changed; recover before adoption'); + validateTransition(before, after); + await validateOwner(after.owner); + if (same(before, after)) return after; + await writeEmbeddingJobRecord(directory, 'rebase.json', { version: 1, before, after }); + await writeEmbeddingJobRecord(directory, 'space.json', after); + await finishIntent(directory); + return after; +} diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 15b2c3f28..20de1e8c6 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -5,7 +5,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; const MAX_RECORD_BYTES = 32 * 1024; -const RECORD_NAMES = new Set(['manifest.json', 'job.json']); +const RECORD_NAMES = new Set(['manifest.json', 'job.json', 'space.json', 'rebase.json']); const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; export async function embeddingJobDirectory(directory: string): Promise { diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 66d9cc54c..c818cc7c5 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -128,11 +128,20 @@ export interface RepoMeta { coveredContent: string; }; /** Checkout-owned durable work. Progress lives in its bounded journal. */ - embeddingJob?: { version: 1; identity: string; graphGeneration: string }; + embeddingJob?: { + version: 1; + identity: string; + graphGeneration: string; + /** Mutable artifact ownership is separate from this graph-bound job. */ + artifactIdentity?: string; + spaceKey?: string; + }; /** Published only after complete coverage, native durability and index validation. */ vectorGeneration?: { version: 1; identity: string; + /** Absent in older separate stores, whose artifact identity equaled the job. */ + artifactIdentity?: string; graphGeneration: string; vectorIdentity: string; inputIdentity: string; diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index 2d8d78b66..a0c45331f 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -55,12 +55,12 @@ export function selectVectorGeneration( kind: 'unavailable', reason: metadata.embeddingJob ? 'embedding-pending' : 'embeddings-unavailable', }; - if (metadata.embeddingJob && metadata.embeddingJob.identity !== vector.identity) - return { kind: 'unavailable', reason: 'embedding-pending' }; + if (metadata.embeddingJob) return { kind: 'unavailable', reason: 'embedding-pending' }; if ( vector.version !== 1 || vector.graphGeneration !== graph.id || !DIGEST.test(vector.identity) || + (vector.artifactIdentity !== undefined && !DIGEST.test(vector.artifactIdentity)) || !DIGEST.test(vector.vectorIdentity) || !DIGEST.test(vector.inputIdentity) || !nonnegative(vector.dimensions) || @@ -76,7 +76,10 @@ export function selectVectorGeneration( return { kind: 'unavailable', reason: 'vector-index-pending' }; return { kind: 'ready', - path: path.join(embeddingGenerationDirectory(metadataDirectory, vector.identity), 'vectors'), + path: path.join( + embeddingGenerationDirectory(metadataDirectory, vector.artifactIdentity ?? vector.identity), + 'vectors', + ), generation: vector, }; } diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index 3a776cd7f..067490107 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -137,6 +137,241 @@ async function fixture(count = 130) { }; } +describe('pending artifacts across graph generations', () => { + it('recovers native sealing before the ownership phase was recorded, then reuses read-only', async () => { + const f = await fixture(5); + const ownership = await import('../../src/storage/embedding-artifact.js'); + const adopt = ownership.adoptEmbeddingArtifact; + const interrupted = vi + .spyOn(ownership, 'adoptEmbeddingArtifact') + .mockImplementation(async (...args) => { + if (args[3].phase === 'sealed') throw new Error('stopped after native seal'); + return adopt(...args); + }); + await expect(f.run()).rejects.toThrow('stopped after native seal'); + interrupted.mockRestore(); + const first = (await loadMeta(f.root))?.embeddingJob; + await f.nextGeneration(); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(5); + expect(result.metadata.vectorGeneration?.artifactIdentity).not.toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('recovers an adopted artifact when metadata persistence fails before any new paid work', async () => { + const f = await fixture(130); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 64) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + metadata: { + ...f.options().metadata, + saveMetadata: async () => { + throw new Error('metadata unavailable'); + }, + }, + }), + ).rejects.toThrow('metadata unavailable'); + expect(f.provider).not.toHaveBeenCalled(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(result.state?.reusedNodes).toBe(64); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + }); + + it('retains the first paid groups across two interrupted rebases with a fresh cursor each time', async () => { + const f = await fixture(130); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 64) throw new Error('first interruption'); + }, + }), + ).rejects.toThrow('first interruption'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(64); + const first = (await loadMeta(f.root))?.embeddingJob; + if (!first?.artifactIdentity) throw new Error('Expected owned pending artifact'); + const file = path.join(f.root, 'embeddings', first.artifactIdentity, 'vectors'); + const inode = (await fs.stat(file)).ino; + const fetch = f.fetch.getMockImplementation(); + if (!fetch) throw new Error('Missing fixture fetch'); + const changed = new Set(['Const:a.ts:n00000']); + f.fetch.mockImplementation(async (label, ids) => + (await fetch(label, ids)).map((node) => + changed.has(node.id) ? { ...node, content: 'const value = 2;' } : node, + ), + ); + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes + state.reusedNodes === 128) + throw new Error('second interruption'); + }, + }), + ).rejects.toThrow('second interruption'); + const second = (await loadMeta(f.root))?.embeddingJob; + expect(second?.identity).not.toBe(first.identity); + expect(second?.artifactIdentity).toBe(first.artifactIdentity); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(65); + changed.add('Const:a.ts:n00001'); + await f.nextGeneration(); + f.provider.mockClear(); + const final = await f.run(); + expect(final.phase).toBe('ready'); + expect(final.state).toMatchObject({ completedNodes: 3, reusedNodes: 127, totalNodes: 130 }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + expect(final.metadata.vectorGeneration?.artifactIdentity).toBe(first.artifactIdentity); + expect(final.metadata.vectorGeneration?.graphGeneration).toBe( + final.metadata.graphGeneration?.id, + ); + expect((await fs.stat(file)).ino).toBe(inode); + const { searchVectorGeneration } = + await import('../../src/core/embeddings/vector-generation-search.js'); + const pool = await import('../../src/core/lbug/pool-adapter.js'); + try { + const hits = await searchVectorGeneration( + { metadataDirectory: f.root, metadata: final.metadata, graphPath: f.options().graphPath }, + async () => [1, 0, 0, 0], + 2, + 1, + async () => f.dependencies.identity(), + ); + expect(hits).toHaveLength(2); + expect(hits?.every((hit) => hit.nodeId.startsWith('Const:a.ts:'))).toBe(true); + } finally { + await pool.closeLbug(); + } + f.provider.mockClear(); + f.openSource.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('retains both pending paid groups and its completed fallback across another graph edit', async () => { + const f = await fixture(130); + const completed = await f.run(); + const fallback = completed.metadata.vectorGeneration; + if (!fallback?.artifactIdentity) throw new Error('Expected completed artifact'); + const fetch = f.fetch.getMockImplementation(); + if (!fetch) throw new Error('Missing fixture fetch'); + const changed = new Set(['Const:a.ts:n00000']); + f.fetch.mockImplementation(async (label, ids) => + (await fetch(label, ids)).map((node) => + changed.has(node.id) ? { ...node, content: 'const value = 2;' } : node, + ), + ); + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes + state.reusedNodes === 64) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + const pending = (await loadMeta(f.root))?.embeddingJob; + expect(pending?.artifactIdentity).not.toBe(fallback.artifactIdentity); + changed.add('Const:a.ts:n00001'); + await f.nextGeneration(); + f.provider.mockClear(); + const final = await f.run(); + expect(final.state).toMatchObject({ completedNodes: 1, reusedNodes: 129 }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(final.metadata.vectorGeneration?.artifactIdentity).toBe(pending?.artifactIdentity); + await expect( + fs.stat(path.join(f.root, 'embeddings', fallback.artifactIdentity, 'vectors')), + ).resolves.toBeDefined(); + }); + + it('uses a sealed but unpublished artifact as a read-only fallback after a graph edit', async () => { + const f = await fixture(5); + const metadata = f.options().metadata; + await expect( + f.run({ + metadata: { + ...metadata, + saveMetadata: async (value) => { + if (value.vectorGeneration) throw new Error('publication interrupted'); + await metadata.saveMetadata(value); + }, + }, + }), + ).rejects.toThrow('publication interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + expect(first?.artifactIdentity).toBeDefined(); + await f.nextGeneration(); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(5); + expect(result.metadata.vectorGeneration?.artifactIdentity).not.toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('returns directly to retained compatible work after using another vector space', async () => { + const f = await fixture(10); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 10) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + f.dependencies.identity.mockReturnValue({ provider: 'local', model: 'other', dimensions: 4 }); + await f.run(); + f.dependencies.identity.mockReturnValue({ provider: 'local', model: 'test', dimensions: 4 }); + f.provider.mockClear(); + f.openSource.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('fails before source/model work when the directly keyed ownership receipt is corrupt', async () => { + const f = await fixture(5); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const pending = (await loadMeta(f.root))?.embeddingJob; + if (!pending?.spaceKey || !pending.artifactIdentity) + throw new Error('Missing ownership receipt'); + const receipt = path.join(f.root, 'embeddings', 'spaces', pending.spaceKey, 'space.json'); + await fs.writeFile(receipt, '{"checksum":"bad","payload":{}}'); + f.provider.mockClear(); + f.openSource.mockClear(); + f.initializeProvider.mockClear(); + await expect(f.run()).rejects.toThrow('checksum mismatch'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + await expect( + fs.stat(path.join(f.root, 'embeddings', pending.artifactIdentity, 'vectors')), + ).resolves.toBeDefined(); + }); +}); + describe('published graph embedding job controller', () => { it('resumes an interrupted bootstrap without rescanning the manifest or losing the pending job', async () => { const f = await fixture(3); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 90fd973bf..94c63193a 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -129,6 +129,122 @@ async function runnerFixture(count: number, content = 'const value = "漢😀";' }; } +describe('rebased pending vector inputs', () => { + it('reuses matching groups in place, relocates lines, recomputes changed inputs and prunes deleted nodes', async () => { + const f = await runnerFixture(2); + const { contentHashForNode } = await import('../../src/core/embeddings/embedding-pipeline.js'); + const { resolveEmbeddingConfig } = await import('../../src/core/embeddings/config.js'); + const ids = ['Const:source.ts:n00000', 'Const:source.ts:n00001', 'Const:source.ts:deleted']; + const nodes = await f.source.fetch('Const', ids); + const hash = contentHashForNode( + nodes[0], + resolveEmbeddingConfig({ subBatchSize: 8, maxInputChars: 1024 }), + ); + for (const [i, id] of ids.entries()) { + const inputHash = i === 0 ? hash : 'previous-input'; + await store.beginNodes([ + { nodeId: id, contentHash: inputHash, sourceStartLine: 9, sourceEndLine: 9 }, + ]); + await store.append(id, inputHash, [{ ...chunk(0), startLine: 9, endLine: 9 }]); + const receipt = await store.finishNode(id, inputHash, 1); + await store.checkpointUnit([receipt]); + } + const inode = (await fs.stat(path.join(f.options.directory, 'vectors'))).ino; + const result = await f.run({ revalidateStoredInputs: true }); + expect(result.state).toMatchObject({ + completedNodes: 1, + reusedNodes: 1, + failedNodes: 0, + chunks: 2, + }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(await store.reusable(ids[2], 'previous-input')).toBeNull(); + const moved = await store.reusable(ids[0], hash); + expect(moved).toMatchObject({ sourceStartLine: 1, sourceEndLine: 1 }); + if (!moved) throw new Error('Matching input was not retained'); + expect(await store.readChunks(moved, 0, 1)).toMatchObject([{ startLine: 1, endLine: 1 }]); + expect((await fs.stat(path.join(f.options.directory, 'vectors'))).ino).toBe(inode); + }); + + it('refuses to prune when current-selection coverage is missing', async () => { + await setup(); + const id = 'Const:source.ts:retained'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await expect(store.pruneUnselected('b'.repeat(64), 1)).rejects.toThrow('no groups pruned'); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + await store.markSelected('b'.repeat(64), [id]); + await expect(store.pruneUnselected('c'.repeat(64), 1)).rejects.toThrow('no groups pruned'); + await store.pruneUnselected('b'.repeat(64), 1); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + }); + + it('prunes an empty new selection and leaves a valid empty artifact', async () => { + await setup(); + const id = 'Const:source.ts:deleted'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.markSelected('b'.repeat(64), [id]); + await store.pruneUnselected('c'.repeat(64), 0); + expect(await store.reusable(id, 'hash')).toBeNull(); + expect(await store.seal(0, 0)).toBe('exact'); + }); + + it('keeps sealed artifacts immutable through every rebase operation', async () => { + await setup(); + const id = 'Const:source.ts:retained'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.seal(1, 1); + expect(store.isSealed).toBe(true); + await expect(store.reuseNode({ nodeId: id, contentHash: 'hash' })).rejects.toThrow('immutable'); + await expect(store.markSelected('b'.repeat(64), [id])).rejects.toThrow('immutable'); + await expect(store.pruneUnselected('b'.repeat(64), 0)).rejects.toThrow('immutable'); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + }); + + it('relocates a multi-batch group without copying or changing its vectors', async () => { + await setup(); + const id = 'Function:source.ts:large'; + await store.beginNodes([ + { nodeId: id, contentHash: 'hash', sourceStartLine: 10, sourceEndLine: 309 }, + ]); + for (let start = 0; start < 300; start += 100) { + await store.append( + id, + 'hash', + Array.from({ length: 100 }, (_, i) => ({ + ...chunk(start + i), + startLine: start + i + 10, + endLine: start + i + 10, + })), + ); + } + await store.finishNode(id, 'hash', 300); + expect(await store.reuseNode({ nodeId: id, contentHash: 'changed' })).toBeNull(); + const receipt = await store.reuseNode({ + nodeId: id, + contentHash: 'hash', + sourceStartLine: 15, + sourceEndLine: 314, + }); + expect(receipt?.chunks).toBe(300); + if (!receipt) throw new Error('Complete group was not retained'); + const first = (await store.readChunks(receipt, 0, 1))[0]; + const last = (await store.readChunks(receipt, 299, 1))[0]; + expect(first).toMatchObject({ startLine: 15, endLine: 15 }); + expect(last).toMatchObject({ startLine: 314, endLine: 314 }); + expect(first.embedding).toEqual(last.embedding); + expect(first.embedding[0]).toBeCloseTo(0.1); + await store.markSelected('b'.repeat(64), [id]); + await store.pruneUnselected('b'.repeat(64), 1); + await store.seal(1, 300); + }); +}); + describe('bounded durable embedding runner', () => { it('refuses a symlinked native shadow sidecar before opening the artifact', async () => { const options = await setup(); diff --git a/gitnexus/test/unit/embedding-artifact.test.ts b/gitnexus/test/unit/embedding-artifact.test.ts new file mode 100644 index 000000000..e65548129 --- /dev/null +++ b/gitnexus/test/unit/embedding-artifact.test.ts @@ -0,0 +1,206 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + adoptEmbeddingArtifact, + readEmbeddingArtifactSpace, + embeddingArtifactSpaceKey, + type EmbeddingArtifactOwner, + type EmbeddingArtifactRecord, +} from '../../src/storage/embedding-artifact.js'; +import * as records from '../../src/storage/embedding-job.js'; + +const space = { + vectorIdentity: 'a'.repeat(64), + inputIdentity: 'b'.repeat(64), + dimensions: 4, + schema: 'gitnexus-vector-store-v2', +}; +const owner = (digit: string): EmbeddingArtifactOwner => ({ + artifactIdentity: 'c'.repeat(64), + jobIdentity: digit.repeat(64), + graphGeneration: digit.repeat(32), + sourceIdentity: digit.repeat(64), + manifestDigest: digit.repeat(64), + phase: 'pending', + revalidateInputs: true, +}); +let root: string; +let validate: ReturnType Promise>>; +const directory = () => path.join(root, 'embeddings', 'spaces', embeddingArtifactSpaceKey(space)); +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-artifact-')); + validate = vi.fn(async () => undefined); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +describe('pending embedding artifact ownership', () => { + it('keeps one artifact through repeated source generations without enumerating job history', async () => { + vi.spyOn(fs, 'readdir').mockRejectedValue(new Error('History scan')); + vi.spyOn(fs, 'opendir').mockRejectedValue(new Error('History scan')); + let record = await readEmbeddingArtifactSpace(root, space, validate); + expect(record).toBeNull(); + for (const digit of ['1', '2', '3']) { + record = await adoptEmbeddingArtifact(root, space, record, owner(digit), validate); + expect(record.owner.artifactIdentity).toBe('c'.repeat(64)); + expect((await readEmbeddingArtifactSpace(root, space, validate))?.owner.jobIdentity).toBe( + digit.repeat(64), + ); + } + expect(fs.readdir).not.toHaveBeenCalled(); + expect(fs.opendir).not.toHaveBeenCalled(); + }); + + it.each(['before-owner', 'after-owner'] as const)( + 'recovers a crash %s without losing ownership', + async (point) => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + if (point === 'before-owner') { + const write = records.writeEmbeddingJobRecord; + vi.spyOn(records, 'writeEmbeddingJobRecord').mockImplementation( + async (dir, name, payload) => { + if (name === 'space.json') throw new Error('Crash before owner'); + await write(dir, name, payload); + }, + ); + } else { + vi.spyOn(fs, 'unlink').mockRejectedValueOnce(new Error('Crash after owner')); + } + await expect( + adoptEmbeddingArtifact(root, space, first, owner('2'), validate), + ).rejects.toThrow('Crash'); + vi.restoreAllMocks(); + const recovered = await readEmbeddingArtifactSpace(root, space, validate); + expect(recovered?.owner).toEqual(owner('2')); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(recovered); + await expect(fs.stat(path.join(directory(), 'rebase.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }, + ); + + it('leaves the old owner unchanged if the new manifest cannot be verified', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + validate.mockRejectedValueOnce(new Error('Missing manifest')); + await expect(adoptEmbeddingArtifact(root, space, first, owner('2'), validate)).rejects.toThrow( + 'Missing manifest', + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('refuses a stale owner instead of overwriting a later adoption', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const second = await adoptEmbeddingArtifact(root, space, first, owner('2'), validate); + await expect(adoptEmbeddingArtifact(root, space, first, owner('3'), validate)).rejects.toThrow( + 'ownership changed', + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(second); + }); + + it('refuses an intent whose before and after owners have both been superseded', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const second: EmbeddingArtifactRecord = { ...first, owner: owner('2') }; + await records.writeEmbeddingJobRecord(directory(), 'rebase.json', { + version: 1, + before: first, + after: second, + }); + await records.writeEmbeddingJobRecord(directory(), 'space.json', { + ...first, + owner: owner('3'), + }); + await expect(readEmbeddingArtifactSpace(root, space, validate)).rejects.toThrow('superseded'); + expect(await records.readEmbeddingJobRecord(directory(), 'rebase.json')).toEqual({ + version: 1, + before: first, + after: second, + }); + }); + + it('requires input revalidation and keeps a pending paid artifact instead of replacing it', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + await expect( + adoptEmbeddingArtifact( + root, + space, + first, + { ...owner('2'), revalidateInputs: false }, + validate, + ), + ).rejects.toThrow('current-input validation'); + await expect( + adoptEmbeddingArtifact( + root, + space, + first, + { ...owner('2'), artifactIdentity: 'd'.repeat(64) }, + validate, + ), + ).rejects.toThrow('retained until sealed'); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('starts a new writer after sealing and retains the completed artifact as its sole fallback', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const sealed = await adoptEmbeddingArtifact( + root, + space, + first, + { ...first.owner, phase: 'sealed' }, + validate, + ); + await expect(adoptEmbeddingArtifact(root, space, sealed, owner('2'), validate)).rejects.toThrow( + 'Rebased artifact', + ); + const next = await adoptEmbeddingArtifact( + root, + space, + sealed, + { + ...owner('2'), + artifactIdentity: 'd'.repeat(64), + fallbackArtifact: first.owner.artifactIdentity, + }, + validate, + ); + expect(next.owner.fallbackArtifact).toBe(first.owner.artifactIdentity); + expect(next.owner.artifactIdentity).not.toBe(first.owner.artifactIdentity); + }); + + it('keeps incompatible input spaces independently owned and directly addressable', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const other = { ...space, inputIdentity: 'd'.repeat(64) }; + expect(await readEmbeddingArtifactSpace(root, other, validate)).toBeNull(); + await adoptEmbeddingArtifact( + root, + other, + null, + { ...owner('2'), artifactIdentity: 'e'.repeat(64) }, + validate, + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('retains corrupt receipts and rejects a symlinked space directory', async () => { + await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const file = path.join(directory(), 'space.json'); + await fs.writeFile(file, '{"checksum":"bad","payload":{}}'); + await expect(readEmbeddingArtifactSpace(root, space, validate)).rejects.toThrow( + 'checksum mismatch', + ); + expect(await fs.readFile(file, 'utf8')).toContain('"bad"'); + const other = { ...space, inputIdentity: 'e'.repeat(64) }; + await fs.symlink( + directory(), + path.join(root, 'embeddings', 'spaces', embeddingArtifactSpaceKey(other)), + process.platform === 'win32' ? 'junction' : 'dir', + ); + await expect(readEmbeddingArtifactSpace(root, other, validate)).rejects.toThrow( + 'owned directory', + ); + }); +}); diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts index 6ccb2f367..81afe3136 100644 --- a/gitnexus/test/unit/vector-generation.test.ts +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -37,6 +37,32 @@ afterEach(async () => { await fs.rm(directory, { recursive: true, force: true }); }); +describe('artifact identity and pending-job routing', () => { + it('routes a rebased artifact by its storage identity while retaining the graph-bound job identity', () => { + const artifactIdentity = 'f'.repeat(64); + const metadata = { ...meta, vectorGeneration: { ...meta.vectorGeneration, artifactIdentity } }; + expect(select(metadata)).toEqual({ + kind: 'ready', + path: path.join('/slot', 'embeddings', artifactIdentity, 'vectors'), + generation: metadata.vectorGeneration, + }); + expect( + select({ + ...metadata, + vectorGeneration: { ...metadata.vectorGeneration, artifactIdentity: '../foreign' }, + }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it('gates a pending job even when its identity equals the last completed pointer', () => { + expect( + select({ ...meta, embeddingJob: { version: 1, identity, graphGeneration: generation } }), + ).toEqual({ + kind: 'unavailable', + reason: 'embedding-pending', + }); + }); +}); + describe('vector generation routing', () => { it.each([ { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }, From a32a4d2f7e336aa2f9f8cc838e3916bc3c7ca134 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 09:47:32 +0000 Subject: [PATCH 20/67] feat(embeddings): preserve historical coverage --- gitnexus/src/core/analysis-progress.ts | 2 +- gitnexus/src/core/analysis-publication.ts | 16 +- gitnexus/src/server/ops-snapshot.ts | 2 +- .../src/storage/embedding-job-progress.ts | 18 +- gitnexus/src/storage/index-capabilities.ts | 7 +- gitnexus/src/storage/repo-meta.ts | 6 + gitnexus/src/storage/vector-generation.ts | 30 +- .../local-backend-preserved-vectors.test.ts | 106 +++++++ .../vector-generation-preservation.test.ts | 264 ++++++++++++++++++ 9 files changed, 441 insertions(+), 10 deletions(-) create mode 100644 gitnexus/test/unit/local-backend-preserved-vectors.test.ts create mode 100644 gitnexus/test/unit/vector-generation-preservation.test.ts diff --git a/gitnexus/src/core/analysis-progress.ts b/gitnexus/src/core/analysis-progress.ts index 9a3659087..fdd75bf78 100644 --- a/gitnexus/src/core/analysis-progress.ts +++ b/gitnexus/src/core/analysis-progress.ts @@ -5,7 +5,7 @@ export type GraphAnalysisPhase = 'pending' | 'building' | 'ready' | 'failed'; /** Counts describe durable acknowledgements. Active work is part of pending, * never an additional category. Totals remain absent before manifest preflight. */ export interface EmbeddingJobProgress { - phase: 'preparing' | 'embedding' | 'indexing' | 'ready' | 'failed' | 'unverified'; + phase: 'preparing' | 'embedding' | 'indexing' | 'ready' | 'preserved' | 'failed' | 'unverified'; totalUnits?: number; completedUnits?: number; reusedUnits?: number; diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts index ad0bb14e3..25492c073 100644 --- a/gitnexus/src/core/analysis-publication.ts +++ b/gitnexus/src/core/analysis-publication.ts @@ -5,7 +5,8 @@ import { constants } from 'node:fs'; import path from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; import type { RepoMeta } from '../storage/repo-meta.js'; -import { INDEX_METADATA_FILE } from '../storage/storage-constants.js'; +import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from '../storage/storage-constants.js'; +import { preserveVectorCoverage } from '../storage/vector-generation.js'; import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; const RECEIPT = 'analysis-publication.json'; @@ -135,7 +136,13 @@ async function writeDurable(directory: string, name: string, bytes: Buffer): Pro async function readSlotMetadata(root: string): Promise { try { - const bytes = await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES); + let bytes: Buffer; + try { + bytes = await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + bytes = await boundedRead(path.join(root, LEGACY_METADATA_FILE), MAX_METADATA_BYTES); + } const metadata: RepoMeta = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); if (!metadata || typeof metadata.repoPath !== 'string') throw new Error('Invalid graph slot metadata; artifacts retained'); @@ -171,12 +178,14 @@ export async function beginInPlaceAnalysis( const prior = await readSlotMetadata(root); if (prior && prior.repoPath !== metadata.repoPath) throw new Error('Graph slot belongs to another repository'); + const attemptId = randomUUID().replaceAll('-', ''); const dirty: RepoMeta = { ...metadata, + vectorGeneration: preserveVectorCoverage(metadata, attemptId), incrementalInProgress: { ...metadata.incrementalInProgress, writePlan: 'in-place', - attemptId: randomUUID().replaceAll('-', ''), + attemptId, }, }; delete dirty.graphGeneration; @@ -234,6 +243,7 @@ export async function prepareAnalysisPublication( const metadataFile = `analysis-meta-${id}.json`; const preparedMeta: RepoMeta = { ...metadata, + vectorGeneration: preserveVectorCoverage(metadata, id), graphPath: graphFile, graphGeneration: { version: 1, id, coveredContent }, }; diff --git a/gitnexus/src/server/ops-snapshot.ts b/gitnexus/src/server/ops-snapshot.ts index f87512208..f29cadaf9 100644 --- a/gitnexus/src/server/ops-snapshot.ts +++ b/gitnexus/src/server/ops-snapshot.ts @@ -20,7 +20,7 @@ const publicEmbeddingProgress = ( ): EmbeddingJobProgress | undefined => { if ( !progress || - !['preparing', 'embedding', 'indexing', 'ready', 'failed', 'unverified'].includes( + !['preparing', 'embedding', 'indexing', 'ready', 'preserved', 'failed', 'unverified'].includes( progress.phase, ) ) diff --git a/gitnexus/src/storage/embedding-job-progress.ts b/gitnexus/src/storage/embedding-job-progress.ts index 6b9d60153..7cb8da501 100644 --- a/gitnexus/src/storage/embedding-job-progress.ts +++ b/gitnexus/src/storage/embedding-job-progress.ts @@ -7,14 +7,28 @@ import { import type { RepoMeta } from './repo-meta.js'; import type { EmbeddingJobState } from './embedding-job-journal.js'; import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js'; -import { embeddingGenerationDirectory } from './vector-generation.js'; +import { embeddingGenerationDirectory, selectVectorGeneration } from './vector-generation.js'; +import { getExactScanLimit } from '../core/platform/capabilities.js'; export async function readEmbeddingJobProgress( metadataDirectory: string, metadata: RepoMeta | null | undefined, ): Promise { const reference = metadata?.embeddingJob; - if (!reference) return undefined; + if (!reference) { + if (!metadata?.vectorGeneration?.preservedForGraph) return undefined; + const selected = selectVectorGeneration( + metadataDirectory, + metadata, + path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'), + getExactScanLimit(), + ); + // These are historical chunks, not completed/reused work in a current + // manifest. Unknown selection totals stay absent, never zero or complete. + return selected.kind === 'ready' && selected.coverage === 'preserved' + ? { phase: 'preserved', chunks: selected.generation.chunks } + : undefined; + } try { if ( reference.version !== 1 || diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts index a959d30f7..b6353473c 100644 --- a/gitnexus/src/storage/index-capabilities.ts +++ b/gitnexus/src/storage/index-capabilities.ts @@ -21,11 +21,16 @@ export async function readIndexCapabilities( exactScanLimit, ); if (selection.kind === 'ready') { - await assertOwnedVectorGeneration(directory, selection.generation.identity); + await assertOwnedVectorGeneration( + directory, + selection.generation.artifactIdentity ?? selection.generation.identity, + ); vectorSearch = { provider: selection.generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', status: selection.generation.indexKind === 'hnsw' ? 'vector-index' : 'exact-scan', exactScanLimit, + coverage: selection.coverage ?? 'current', + sourceGraphGeneration: selection.generation.graphGeneration, }; } else { vectorSearch = { diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index c818cc7c5..3ebc18e25 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -143,6 +143,9 @@ export interface RepoMeta { /** Absent in older separate stores, whose artifact identity equaled the job. */ artifactIdentity?: string; graphGeneration: string; + /** Explicit permission to serve this completed historical artifact against + * a newer graph. graphGeneration remains its original coverage receipt. */ + preservedForGraph?: string; vectorIdentity: string; inputIdentity: string; dimensions: number; @@ -279,6 +282,9 @@ export interface RepoMeta { status: 'vector-index' | 'exact-scan' | 'unavailable'; exactScanLimit: number; reason?: string; + /** Native availability and coverage of the current selection differ. */ + coverage?: 'current' | 'preserved' | 'unknown'; + sourceGraphGeneration?: string; }; }; /** diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index a0c45331f..6319cd6f5 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -7,7 +7,7 @@ export type PublishedVectorGeneration = NonNullable @@ -29,6 +29,29 @@ export function metadataMatchesGraph( return path.resolve(metadataDirectory, recorded ?? 'lbug') === path.resolve(graphPath); } +/** Advance only a previously bound completed reference. Called under the slot + * lock before invalidating an in-place graph, or while preparing a staged one. + * It neither opens vectors nor upgrades missing/foreign coverage provenance. */ +export function preserveVectorCoverage( + metadata: RepoMeta, + nextGeneration: string, +): RepoMeta['vectorGeneration'] { + if (!GENERATION.test(nextGeneration)) throw new Error('Invalid preservation target'); + const vector = metadata.vectorGeneration; + const graph = metadata.graphGeneration; + if ( + !vector || + vector.version !== 1 || + !GENERATION.test(vector.graphGeneration) || + !graph || + graph.version !== 1 || + !GENERATION.test(graph.id) || + (vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id) + ) + return vector; + return { ...vector, preservedForGraph: nextGeneration }; +} + /** Run before model initialization, vector COUNTs, or opening an artifact. */ export function selectVectorGeneration( metadataDirectory: string, @@ -58,7 +81,9 @@ export function selectVectorGeneration( if (metadata.embeddingJob) return { kind: 'unavailable', reason: 'embedding-pending' }; if ( vector.version !== 1 || - vector.graphGeneration !== graph.id || + !GENERATION.test(vector.graphGeneration) || + (vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id) || + (vector.preservedForGraph !== undefined && !GENERATION.test(vector.preservedForGraph)) || !DIGEST.test(vector.identity) || (vector.artifactIdentity !== undefined && !DIGEST.test(vector.artifactIdentity)) || !DIGEST.test(vector.vectorIdentity) || @@ -81,6 +106,7 @@ export function selectVectorGeneration( 'vectors', ), generation: vector, + ...(vector.graphGeneration === graph.id ? {} : { coverage: 'preserved' as const }), }; } diff --git a/gitnexus/test/unit/local-backend-preserved-vectors.test.ts b/gitnexus/test/unit/local-backend-preserved-vectors.test.ts new file mode 100644 index 000000000..4fed67426 --- /dev/null +++ b/gitnexus/test/unit/local-backend-preserved-vectors.test.ts @@ -0,0 +1,106 @@ +import path from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const mocks = vi.hoisted(() => ({ metadata: vi.fn(), search: vi.fn(), hydrate: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', async (original) => ({ + ...(await original()), + loadMeta: mocks.metadata, +})); +vi.mock('../../src/core/lbug/pool-adapter.js', async (original) => ({ + ...(await original()), + executeParameterized: mocks.hydrate, +})); +vi.mock('../../src/core/embeddings/vector-generation-search.js', () => ({ + searchVectorGeneration: mocks.search, +})); +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const current = 'a'.repeat(32); +const source = 'b'.repeat(32); +const graph = path.join('/repo/.gitnexus', `lbug.generation.${current}`); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'new', + indexedAt: '', + graphPath: path.basename(graph), + graphGeneration: { version: 1, id: current, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity: 'd'.repeat(64), + artifactIdentity: 'e'.repeat(64), + graphGeneration: source, + preservedForGraph: current, + vectorIdentity: 'f'.repeat(64), + inputIdentity: '0'.repeat(64), + dimensions: 4, + nodes: 3, + chunks: 3, + indexKind: 'exact', + }, +}; +interface Reader { + semanticSearch(repo: { lbugPath: string }, query: string, limit: number): Promise; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.metadata.mockResolvedValue(metadata); + mocks.search.mockResolvedValue( + ['deleted', 'kept', 'second'].map((name, i) => ({ + nodeId: `Function:a.ts:${name}`, + chunkIndex: 0, + startLine: 1, + endLine: 2, + distance: i / 10, + })), + ); + mocks.hydrate.mockImplementation(async (file, _query, { nodeId }) => { + expect(file).toBe(graph); + return nodeId.endsWith(':deleted') + ? [] + : [{ name: nodeId.split(':').at(-1), filePath: 'current/a.ts' }]; + }); +}); + +describe('preserved semantic hits', () => { + it('hydrates only the current graph, drops deleted IDs and keeps surviving rank order', async () => { + const backend = new LocalBackend() as unknown as Reader; + const hits = await backend.semanticSearch({ lbugPath: graph }, 'example', 3); + expect(hits).toEqual([ + { + nodeId: 'Function:a.ts:kept', + name: 'kept', + type: 'Function', + filePath: 'current/a.ts', + distance: 0.1, + startLine: 1, + endLine: 2, + }, + { + nodeId: 'Function:a.ts:second', + name: 'second', + type: 'Function', + filePath: 'current/a.ts', + distance: 0.2, + startLine: 1, + endLine: 2, + }, + ]); + expect(mocks.search.mock.calls[0][0]).toEqual({ + metadataDirectory: path.dirname(graph), + metadata, + graphPath: graph, + }); + expect(mocks.hydrate).toHaveBeenCalledTimes(3); + }); + it('does not open the old vector artifact when a new job is pending', async () => { + mocks.metadata.mockResolvedValue({ + ...metadata, + embeddingJob: { version: 1, identity: '1'.repeat(64), graphGeneration: current }, + }); + const backend = new LocalBackend() as unknown as Reader; + expect(await backend.semanticSearch({ lbugPath: graph }, 'example', 3)).toEqual([]); + expect(mocks.search).not.toHaveBeenCalled(); + expect(mocks.hydrate).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/vector-generation-preservation.test.ts b/gitnexus/test/unit/vector-generation-preservation.test.ts new file mode 100644 index 000000000..650d3e7f8 --- /dev/null +++ b/gitnexus/test/unit/vector-generation-preservation.test.ts @@ -0,0 +1,264 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { selectVectorGeneration } from '../../src/storage/vector-generation.js'; +import { readIndexCapabilities } from '../../src/storage/index-capabilities.js'; +import { readEmbeddingJobProgress } from '../../src/storage/embedding-job-progress.js'; +import { publicOpsProgress } from '../../src/server/ops-snapshot.js'; +import type { AnalyzeJobProgress } from '../../src/server/analyze-job.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const generation = 'a'.repeat(32); +const identity = 'b'.repeat(64); +const artifactIdentity = 'c'.repeat(64); +const roots: string[] = []; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-preserved-'))); + roots.push(root); + const graphPath = `lbug.generation.${generation}`; + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'old', + indexedAt: '', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + graphPath, + graphGeneration: { version: 1, id: generation, coveredContent: 'd'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + artifactIdentity, + graphGeneration: generation, + vectorIdentity: 'e'.repeat(64), + inputIdentity: 'f'.repeat(64), + dimensions: 4, + nodes: 2, + chunks: 3, + indexKind: 'exact', + }, + }; + const graph = path.join(root, graphPath); + const vectors = path.join(root, 'embeddings', artifactIdentity, 'vectors'); + await fs.mkdir(path.dirname(vectors), { recursive: true }); + await fs.writeFile(vectors, 'untouched vector artifact'); + await fs.writeFile(graph, 'graph'); + await saveMeta(root, metadata); + const actions = { verify: async () => {}, saveMetadata: (m: RepoMeta) => saveMeta(root, m) }; + const stage = async () => { + const file = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(file, 'next graph'); + return file; + }; + return { root, metadata, graph, vectors, actions, stage }; +} +afterEach(async () => { + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('preserved vector coverage', () => { + it('advances two staged graphs with a small historical reference and no vector writes', async () => { + const f = await fixture(); + const before = await fs.stat(f.vectors); + for (let i = 0; i < 2; i++) { + const prior = await loadMeta(f.root); + if (!prior) throw new Error('Missing metadata'); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + prior, + '1'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toMatchObject({ + identity, + artifactIdentity, + graphGeneration: generation, + preservedForGraph: receipt.id, + }); + expect( + selectVectorGeneration(f.root, published, path.join(f.root, receipt.graphFile), 10000), + ).toMatchObject({ kind: 'ready', path: f.vectors, coverage: 'preserved' }); + expect(await readIndexCapabilities(f.root, published)).toMatchObject({ + vectorSearch: { + status: 'exact-scan', + coverage: 'preserved', + sourceGraphGeneration: generation, + }, + }); + expect(await readEmbeddingJobProgress(f.root, published)).toEqual({ + phase: 'preserved', + chunks: 3, + }); + } + const after = await fs.stat(f.vectors); + expect([after.ino, after.size, after.mtimeMs]).toEqual([ + before.ino, + before.size, + before.mtimeMs, + ]); + expect(await fs.readFile(f.vectors, 'utf8')).toBe('untouched vector artifact'); + }); + + it('binds preservation before invalidating an in-place graph and retains its inode', async () => { + const f = await fixture(); + const inode = (await fs.stat(f.graph)).ino; + const dirty = await beginInPlaceAnalysis( + f.root, + { + ...f.metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + f.actions.saveMetadata, + ); + expect(selectVectorGeneration(f.root, dirty, f.graph, 10000)).toMatchObject({ + kind: 'unavailable', + }); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Missing attempt'); + const receipt = await prepareAnalysisPublication( + f.root, + f.graph, + { + ...dirty, + incrementalInProgress: undefined, + }, + '2'.repeat(64), + { mode: 'in-place', attemptId }, + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toMatchObject({ + graphGeneration: generation, + preservedForGraph: receipt.id, + }); + expect(selectVectorGeneration(f.root, published, f.graph, 10000)).toMatchObject({ + kind: 'ready', + coverage: 'preserved', + }); + expect((await fs.stat(f.graph)).ino).toBe(inode); + }); + + it('does not turn an unrelated generation into preserved coverage', async () => { + const f = await fixture(); + const vector = f.metadata.vectorGeneration; + if (!vector) throw new Error('Missing vector fixture'); + const invalid = { + ...f.metadata, + vectorGeneration: { ...vector, graphGeneration: 'f'.repeat(32) }, + }; + await saveMeta(f.root, invalid); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + invalid, + '3'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect( + selectVectorGeneration(f.root, published, path.join(f.root, receipt.graphFile), 10000), + ).toMatchObject({ kind: 'unavailable', reason: 'vector-generation-mismatch' }); + }); + + it('reports a rebased current artifact by artifact identity, not the job directory', async () => { + const f = await fixture(); + expect(await readIndexCapabilities(f.root, f.metadata)).toMatchObject({ + vectorSearch: { + status: 'exact-scan', + coverage: 'current', + sourceGraphGeneration: generation, + }, + }); + }); + + it('gates preserved coverage while a job is pending and rejects a foreign preservation target', async () => { + const f = await fixture(); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + f.metadata, + '4'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + if (!published) throw new Error('Missing publication'); + const graph = path.join(f.root, receipt.graphFile); + expect( + selectVectorGeneration( + f.root, + { ...published, embeddingJob: { version: 1, identity, graphGeneration: receipt.id } }, + graph, + 10000, + ), + ).toEqual({ kind: 'unavailable', reason: 'embedding-pending' }); + expect( + selectVectorGeneration( + f.root, + { + ...published, + graphGeneration: { version: 1, id: '9'.repeat(32), coveredContent: '4'.repeat(64) }, + }, + graph, + 10000, + ), + ).toEqual({ kind: 'unavailable', reason: 'vector-generation-mismatch' }); + }); + + it('forwards preserved progress without inventing current selection totals or leaking identifiers', () => { + const projected = publicOpsProgress({ + phase: 'graph-ready', + percent: 100, + message: 'Graph ready', + embeddingProgress: { + phase: 'preserved', + chunks: 3, + artifactIdentity, + sourcePath: '/private', + }, + } as AnalyzeJobProgress); + expect(projected.embeddingProgress).toEqual({ phase: 'preserved', chunks: 3 }); + }); +}); + +describe('legacy-only metadata publication', () => { + it('does not hide corrupt canonical metadata behind an older legacy mirror', async () => { + const f = await fixture(); + await fs.writeFile(path.join(f.root, 'gitnexus.json'), '{broken'); + await expect( + prepareAnalysisPublication(f.root, await f.stage(), f.metadata, '7'.repeat(64)), + ).rejects.toThrow(); + await expect(fs.stat(path.join(f.root, 'analysis-publication.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('binds the legacy metadata receipt and rejects a superseding legacy writer', async () => { + const f = await fixture(); + await fs.unlink(path.join(f.root, 'gitnexus.json')); + await prepareAnalysisPublication(f.root, await f.stage(), f.metadata, '5'.repeat(64)); + await fs.writeFile( + path.join(f.root, 'meta.json'), + JSON.stringify({ ...f.metadata, lastCommit: 'newer' }), + ); + await expect(publishPreparedAnalysis(f.root, f.actions)).rejects.toThrow('superseded'); + expect((await loadMeta(f.root))?.lastCommit).toBe('newer'); + }); + it('accepts an unchanged legacy receipt without filling in unknown embedding provenance', async () => { + const f = await fixture(); + const legacy = { + ...f.metadata, + graphGeneration: undefined, + vectorGeneration: undefined, + embeddingDims: 4, + }; + await saveMeta(f.root, legacy); + await fs.unlink(path.join(f.root, 'gitnexus.json')); + await prepareAnalysisPublication(f.root, await f.stage(), legacy, '6'.repeat(64)); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toBeUndefined(); + expect(published?.embeddingDims).toBe(4); + }); +}); From 8854eb887326cc7b77ce7f0a4168d23e110aab78 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 10:46:52 +0000 Subject: [PATCH 21/67] feat(embeddings): reuse verified legacy vectors Retain a directly referenced legacy source across pending-artifact rebases. Stream checkpoint exclusions and build a native metadata index without copying vectors or retaining corpus-sized JavaScript maps. Require recorded provider identity and provable complete input groups. Keep unknown provenance explicit, source artifacts immutable, compatible relocations free of provider calls, and new inputs in normal batches. The controller remains inactive until coordinated writer activation. Refs #3598 --- .../core/embeddings/embedding-job-runner.ts | 37 +- gitnexus/src/core/embeddings/embedding-job.ts | 75 +++- .../src/core/lbug/legacy-vector-source.ts | 418 ++++++++++++++++++ gitnexus/src/core/lbug/vector-store.ts | 129 ++++++ gitnexus/src/storage/embedding-artifact.ts | 13 +- gitnexus/src/storage/embedding-job.ts | 8 +- .../src/storage/legacy-embedding-source.ts | 290 ++++++++++++ gitnexus/src/storage/repo-meta.ts | 3 + .../legacy-embedding-controller.test.ts | 305 +++++++++++++ .../legacy-vector-reuse-index.test.ts | 135 ++++++ .../integration/legacy-vector-source.test.ts | 264 +++++++++++ .../test/unit/legacy-embedding-source.test.ts | 197 +++++++++ 12 files changed, 1860 insertions(+), 14 deletions(-) create mode 100644 gitnexus/src/core/lbug/legacy-vector-source.ts create mode 100644 gitnexus/src/storage/legacy-embedding-source.ts create mode 100644 gitnexus/test/integration/legacy-embedding-controller.test.ts create mode 100644 gitnexus/test/integration/legacy-vector-reuse-index.test.ts create mode 100644 gitnexus/test/integration/legacy-vector-source.test.ts create mode 100644 gitnexus/test/unit/legacy-embedding-source.test.ts diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index c6f949f91..ba42d52a7 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -18,9 +18,26 @@ import { iterateNodeChunks } from './chunker.js'; import { generateEmbeddingText, splitEmbeddingText } from './text-generator.js'; import { extractStructuralNames } from './structural-extractor.js'; import { resolveEmbeddingConfig } from './config.js'; -import { isShortLabel, STRUCTURAL_LABELS, type EmbeddingConfig } from './types.js'; +import { + isShortLabel, + STRUCTURAL_LABELS, + type EmbeddingConfig, + type EmbeddableNode, +} from './types.js'; import type { embedBatch } from './embedder.js'; +export interface EmbeddingReuseSource { + options: { batchBytes: number }; + requiresInputValidation?: boolean; + reusable( + nodeId: string, + contentHash: string, + node: EmbeddableNode, + prepareInputValidation?: () => Promise, + ): Promise; + readChunks: VectorStore['readChunks']; +} + export interface EmbeddingJobRunnerOptions { directory: string; manifest: EmbeddingManifestReceipt; @@ -28,7 +45,7 @@ export interface EmbeddingJobRunnerOptions { journal: EmbeddingJobJournal; store: VectorStore; /** The controller has validated vector/input identities before providing reuse. */ - reuse?: Pick; + reuse?: EmbeddingReuseSource; /** The writer was adopted from an older graph. Persist this choice for the * whole job: old complete receipts cannot skip current-input validation. */ revalidateStoredInputs?: boolean; @@ -201,6 +218,15 @@ export async function runEmbeddingJob( } }; + const prepareInputValidation = options.reuse?.requiresInputValidation + ? async () => { + // Release a provider batch only when a matching legacy group needs + // input expansion. Ineligible groups keep normal provider batching. + await flush(); + if (abort) throw abort; + } + : undefined; + try { // Hash once per fetched node; current-job complete groups never fetch/hash. const inputs = fetched.nodes.map((node) => ({ @@ -228,7 +254,12 @@ export async function runEmbeddingJob( signal?.throwIfAborted(); if (options.revalidateStoredInputs && complete.has(node.id)) continue; const reuse = options.reuse; - const reusable = await reuse?.reusable(node.id, input.contentHash); + const reusable = await reuse?.reusable( + node.id, + input.contentHash, + node, + prepareInputValidation, + ); if (reusable && reuse) { // Flush before copy so there is only one vector batch in memory. await flush(); diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 196dcbd77..72aa49c9a 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -21,6 +21,8 @@ import { import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; +import { LegacyVectorSource, legacyVectorCompatibility } from '../lbug/legacy-vector-source.js'; +import { readLegacyEmbeddingSource } from '../../storage/legacy-embedding-source.js'; import { drainAnalysisNativeResources } from '../lbug/lbug-adapter.js'; import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; @@ -245,6 +247,29 @@ export async function runPublishedEmbeddingJob( throw new Error('Owned embedding artifact manifest changed; paid artifacts retained'); }; let artifact = await readEmbeddingArtifactSpace(root, space, validateOwner); + // A direct legacy receipt survives graph-bound journal replacement. Prefer + // the newer complete artifact when one exists; never open two reuse readers. + const legacyIdentity = + artifact?.owner.fallbackLegacy ?? + (!artifact && !previousCompatible ? metadata.legacyVectorSource : undefined); + let legacy = legacyIdentity ? await readLegacyEmbeddingSource(root, legacyIdentity) : undefined; + if (legacy) { + const incompatible = + legacyVectorCompatibility(legacy.recordedIdentity, vector) ?? + (legacy.recordedInputIdentity !== undefined && + legacy.recordedInputIdentity !== identities.inputIdentity + ? 'Legacy embedding input provenance differs from this request; source retained.' + : undefined); + if (incompatible) { + if (artifact?.owner.fallbackLegacy) throw new Error(incompatible); + options.log?.(incompatible); + legacy = undefined; + } else { + const current = await fs.lstat(options.graphPath, { bigint: true }); + if (String(current.dev) === legacy.file.dev && String(current.ino) === legacy.file.ino) + throw new Error('Legacy reuse requires a retired source distinct from the mutable graph'); + } + } if ( !artifact && metadata.embeddingJob && @@ -258,16 +283,17 @@ export async function runPublishedEmbeddingJob( artifact ? artifact.owner.phase === 'sealed' ? artifact.owner.jobIdentity !== identities.identity - : Boolean(artifact.owner.fallbackArtifact) + : Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy) : Boolean( - previousCompatible && - (previous.artifactIdentity ?? previous.identity) !== identities.identity, + legacy || + (previousCompatible && + (previous.artifactIdentity ?? previous.identity) !== identities.identity), ); limits(needsFallback()); const directory = await jobDirectory(root, identities.identity); let source: GraphSource | undefined; let store: VectorStore | undefined; - let reuse: VectorStore | undefined; + let reuse: VectorStore | LegacyVectorSource | undefined; let runtime: EmbeddingSidecarReservation | undefined; try { if (usesLocalRuntime) { @@ -290,7 +316,7 @@ export async function runPublishedEmbeddingJob( recoverEmptyBootstrap: !(await existsOwnedFile( path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json'), )), - ...limits(Boolean(artifact.owner.fallbackArtifact)), + ...limits(Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy)), }); if (store.isSealed) { await store.close(); @@ -299,7 +325,12 @@ export async function runPublishedEmbeddingJob( root, space, artifact, - { ...artifact.owner, phase: 'sealed', fallbackArtifact: undefined }, + { + ...artifact.owner, + phase: 'sealed', + fallbackArtifact: undefined, + fallbackLegacy: undefined, + }, validateOwner, ); } @@ -329,6 +360,7 @@ export async function runPublishedEmbeddingJob( artifactIdentity: string, revalidateInputs: boolean, fallbackArtifact?: string, + fallbackLegacy?: string, ): EmbeddingArtifactOwner => ({ artifactIdentity, jobIdentity: identities.identity, @@ -338,6 +370,7 @@ export async function runPublishedEmbeddingJob( phase: 'pending', revalidateInputs, ...(fallbackArtifact === undefined ? {} : { fallbackArtifact }), + ...(fallbackLegacy === undefined ? {} : { fallbackLegacy }), }); let owner: EmbeddingArtifactOwner; if (artifact?.owner.phase === 'sealed' && artifact.owner.jobIdentity === identities.identity) { @@ -351,6 +384,7 @@ export async function runPublishedEmbeddingJob( artifact.owner.artifactIdentity, artifact.owner.revalidateInputs || artifact.owner.jobIdentity !== identities.identity, artifact.owner.fallbackArtifact, + artifact.owner.fallbackLegacy, ); } else { // Start a space beside its first job. This also adopts an older bounded @@ -361,6 +395,7 @@ export async function runPublishedEmbeddingJob( identities.identity, false, previousCompatible ? (previous.artifactIdentity ?? previous.identity) : undefined, + legacy?.identity, ); if (owner.fallbackArtifact === owner.artifactIdentity) owner.fallbackArtifact = undefined; if ( @@ -371,7 +406,7 @@ export async function runPublishedEmbeddingJob( owner.phase = 'sealed'; } artifact = await adoptEmbeddingArtifact(root, space, artifact, owner, validateOwner); - const storeResources = limits(Boolean(owner.fallbackArtifact)); + const storeResources = limits(Boolean(owner.fallbackArtifact || owner.fallbackLegacy)); if ( metadata.embeddingJob?.identity !== identities.identity || metadata.embeddingJob.artifactIdentity !== owner.artifactIdentity || @@ -414,6 +449,25 @@ export async function runPublishedEmbeddingJob( mode: 'read', ...storeResources, }); + } else if (owner.fallbackLegacy && !store.isSealed) { + if (!legacy || legacy.identity !== owner.fallbackLegacy) + throw new Error('Legacy source does not match the owned fallback; artifacts retained'); + reuse = await LegacyVectorSource.open( + { + path: legacy.path, + sourceIdentity: legacy.identity, + recordedIdentity: legacy.recordedIdentity, + expectedIdentity: vector, + recordedInputIdentity: legacy.recordedInputIdentity, + expectedInputIdentity: identities.inputIdentity, + config, + signal, + log: options.log, + ...storeResources, + }, + store, + legacy.excludedIds, + ); } const provider: typeof embedBatch = dependencies.provider ?? @@ -468,7 +522,12 @@ export async function runPublishedEmbeddingJob( root, space, artifact, - { ...artifact.owner, phase: 'sealed', fallbackArtifact: undefined }, + { + ...artifact.owner, + phase: 'sealed', + fallbackArtifact: undefined, + fallbackLegacy: undefined, + }, validateOwner, ); } diff --git a/gitnexus/src/core/lbug/legacy-vector-source.ts b/gitnexus/src/core/lbug/legacy-vector-source.ts new file mode 100644 index 000000000..2978b4aea --- /dev/null +++ b/gitnexus/src/core/lbug/legacy-vector-source.ts @@ -0,0 +1,418 @@ +/** Bounded, read-only reuse of verified legacy monolithic embedding groups. */ +import lbug, { type LbugValue } from '@ladybugdb/core'; +import fs from 'node:fs/promises'; +import { + openLbugConnection, + closeLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; +import { + VectorStore, + type LegacyVectorGroup, + type VectorNodeReceipt, + type StoredVectorChunk, +} from './vector-store.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import type { EmbeddingIdentity } from '../embeddings/embedding-identity.js'; +import { resolveEmbeddingConfig } from '../embeddings/config.js'; +import { iterateNodeChunks } from '../embeddings/chunker.js'; +import { generateEmbeddingText, splitEmbeddingText } from '../embeddings/text-generator.js'; +import { extractStructuralNames } from '../embeddings/structural-extractor.js'; +import { + isShortLabel, + STRUCTURAL_LABELS, + type EmbeddableNode, + type EmbeddingConfig, +} from '../embeddings/types.js'; + +export interface LegacyVectorSourceOptions { + /** Caller validates ownership and holds the slot lock. WAL recovery and + * checkpointing must already have completed in the graph process. */ + path: string; + sourceIdentity: string; + recordedIdentity?: Partial; + expectedIdentity: EmbeddingIdentity; + /** Optional historical input receipt; never synthesized from current config. */ + recordedInputIdentity?: string; + expectedInputIdentity: string; + config?: Partial; + bufferPoolBytes: number; + threads: number; + batchBytes: number; + signal?: AbortSignal; + log?: (message: string) => void; +} + +/** A row hash does not identify its provider. Never fill missing historical + * fields from today's configuration, including for a mere file relocation. */ +export function legacyVectorCompatibility( + recorded: Partial | undefined, + expected: EmbeddingIdentity, +): string | undefined { + if ( + !recorded || + typeof recorded.provider !== 'string' || + !/^(local|http:[a-f0-9]{64})$/.test(recorded.provider) || + typeof recorded.model !== 'string' || + !recorded.model || + !Number.isSafeInteger(recorded.dimensions) || + !recorded.dimensions || + recorded.dimensions < 1 + ) + return 'Legacy embedding provider/model/dimensions provenance is unknown; source retained.'; + if ( + recorded.provider !== expected.provider || + recorded.model !== expected.model || + recorded.dimensions !== expected.dimensions + ) + return 'Legacy embedding provider/model/dimensions differ from this request; source retained.'; + return undefined; +} + +const count = (value: unknown): number => { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid legacy vector count'); + const n = Number(value); + if (!Number.isSafeInteger(n) || n < 0) throw new Error('Invalid legacy vector count'); + return n; +}; +type Row = Record; +function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult { + if (!Array.isArray(result)) return result; + if (result.length === 1) return result[0]; + for (const part of result) part.close(); + throw new Error('Unexpected legacy query result count'); +} + +export class LegacyVectorSource { + readonly requiresInputValidation = true; + private readonly statements = new Map(); + private readonly config: EmbeddingConfig; + private active = false; + private closed = false; + private closing?: Promise; + private constructor( + readonly options: LegacyVectorSourceOptions, + private readonly handle: LbugConnectionHandle, + private readonly index: VectorStore, + ) { + this.config = resolveEmbeddingConfig(options.config); + } + + static async open( + options: LegacyVectorSourceOptions, + index: VectorStore, + excluded: () => AsyncIterable | Iterable, + ): Promise { + const incompatible = legacyVectorCompatibility( + options.recordedIdentity, + options.expectedIdentity, + ); + if (incompatible) throw new Error(incompatible); + if ( + !/^[a-f0-9]{64}$/.test(options.sourceIdentity) || + !/^[a-f0-9]{64}$/.test(options.expectedInputIdentity) || + options.expectedIdentity.dimensions !== index.options.dimensions || + !Number.isSafeInteger(options.batchBytes) || + options.batchBytes < 1024 + ) + throw new Error('Invalid legacy vector reuse reservation'); + if ( + options.recordedInputIdentity !== undefined && + options.recordedInputIdentity !== options.expectedInputIdentity + ) + throw new Error( + 'Legacy embedding input provenance differs from this request; source retained.', + ); + if (!options.recordedInputIdentity) + options.log?.( + 'Legacy multi-chunk input provenance is unknown (chunk size/overlap); source retained and those groups require regeneration.', + ); + options.signal?.throwIfAborted(); + const stat = await fs.lstat(options.path); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy vectors require an owned regular database'); + const source = new LegacyVectorSource( + options, + await openLbugConnection(lbug, options.path, { + readOnly: true, + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }), + index, + ); + try { + await index.prepareLegacyReuseIndex( + options.sourceIdentity, + () => source.groups(), + excluded, + options.signal, + ); + return source; + } catch (error) { + await source.close(); + throw error; + } + } + + private acquire(): void { + if (this.active || this.closed) throw new Error('Legacy vector source is closed or active'); + this.options.signal?.throwIfAborted(); + this.active = true; + } + private async result(sql: string, params?: Record): Promise { + if (!params) return single(await this.handle.conn.query(sql)); + let prepared = this.statements.get(sql); + if (!prepared) { + prepared = await this.handle.conn.prepare(sql); + if (!prepared.isSuccess()) throw new Error(prepared.getErrorMessage()); + this.statements.set(sql, prepared); + } + return single(await this.handle.conn.execute(prepared, params)); + } + + private async *groups(): AsyncGenerator { + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + // Native sorting is charged to this explicit pool; JS retains one group + // and never fetches vector payloads while building the receipt index. + result = await this.result( + 'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash, e.startLine AS startLine, e.endLine AS endLine ORDER BY e.nodeId, e.chunkIndex', + ); + let group: LegacyVectorGroup | undefined; + let valid = true; + while (result.hasNext()) { + this.options.signal?.throwIfAborted(); + const row = result.getNextSync(); + if (typeof row.nodeId !== 'string') throw new Error('Invalid legacy node identity'); + assertValidGraphNodeId(row.nodeId, 'Legacy vector source'); + if (group?.nodeId !== row.nodeId) { + if (group && valid) yield group; + group = { + nodeId: row.nodeId, + contentHash: typeof row.contentHash === 'string' ? row.contentHash : '', + chunks: 0, + sourceStartLine: count(row.startLine), + sourceEndLine: count(row.endLine), + }; + valid = Boolean(group.contentHash); + } + if (!group) throw new Error('Missing legacy group'); + const ordinal = count(row.chunkIndex), + start = count(row.startLine), + end = count(row.endLine); + if ( + ordinal !== group.chunks || + row.id !== `${group.nodeId}:${ordinal}` || + row.contentHash !== group.contentHash || + end < start + ) + valid = false; + group.chunks++; + group.sourceEndLine = Math.max(group.sourceEndLine, end); + } + if (group && valid) yield group; + } finally { + try { + result?.close(); + } finally { + this.active = false; + } + } + } + + private async rows( + receipt: VectorNodeReceipt, + start: number, + length: number, + vectors: boolean, + ): Promise { + const result = await this.result( + `UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.contentHash AS contentHash${vectors ? ', e.embedding AS embedding' : ''}`, + { + ids: Array.from({ length }, (_, i) => `${receipt.nodeId}:${start + i}`), + }, + ); + const rows: Row[] = []; + try { + while (result.hasNext()) { + if (rows.length >= length) throw new Error('Legacy query exceeded its bounded result size'); + rows.push(result.getNextSync()); + } + } finally { + result.close(); + } + rows.sort((a, b) => count(a.chunkIndex) - count(b.chunkIndex)); + if ( + rows.length !== length || + rows.some( + (row, i) => + row.nodeId !== receipt.nodeId || + row.contentHash !== receipt.contentHash || + count(row.chunkIndex) !== start + i, + ) + ) + throw new Error('Legacy complete group changed; source retained'); + return rows; + } + + /** Legacy hashes did not record chunk-size/overlap. Multiple chunks may have + * identical line spans but different character boundaries, so their layout + * cannot replace an input receipt. Short labels ignore those settings; a + * complete single-chunk group covers its full input without overlap. */ + async reusable( + nodeId: string, + contentHash: string, + node: EmbeddableNode, + prepareInputValidation?: () => Promise, + ): Promise { + this.acquire(); + const originalMethods = node.methodNames, + originalFields = node.fieldNames; + try { + const group = await this.index.legacyReuseGroup(this.options.sourceIdentity, nodeId); + if (!group || group.contentHash !== contentHash || node.id !== nodeId) return null; + if (!this.options.recordedInputIdentity && !isShortLabel(node.label) && group.chunks > 1) { + return null; + } + await prepareInputValidation?.(); + const receipt: VectorNodeReceipt = { ...group, complete: true }; + const rowBytes = 512 + 8 * nodeId.length; + if (rowBytes > this.options.batchBytes) + throw new Error('Legacy metadata does not fit its reservation'); + const blockSize = Math.min( + 64, + Math.max(1, Math.floor(Math.min(32768, this.options.batchBytes) / rowBytes)), + ); + let block: Row[] = [], + ordinal = 0, + delta: number | undefined; + for await (const span of this.inputSpans(node)) { + if (ordinal >= group.chunks) return null; + if (ordinal % blockSize === 0) + block = await this.rows( + receipt, + ordinal, + Math.min(blockSize, group.chunks - ordinal), + false, + ); + const row = block[ordinal % blockSize]; + const start = count(row.startLine), + end = count(row.endLine); + delta ??= span.startLine - start; + if (start + delta !== span.startLine || end + delta !== span.endLine) return null; + ordinal++; + } + if (ordinal !== group.chunks || delta === undefined || (node.startLine ?? 0) - delta < 0) + return null; + // Legacy rows recorded chunk ranges, not the original node's end. + // Do not let the copy path clamp the final chunk to an invented range. + return { + ...receipt, + sourceStartLine: (node.startLine ?? 0) - delta, + sourceEndLine: undefined, + }; + } finally { + node.methodNames = originalMethods; + node.fieldNames = originalFields; + this.active = false; + } + } + + private async *inputSpans( + node: EmbeddableNode, + ): AsyncGenerator<{ startLine: number; endLine: number }> { + if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames(node.content, node.filePath); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + /* Same optional enrichment and text-version contract as generation. */ + } + } + const chunks = isShortLabel(node.label) + ? [ + { + text: node.content, + chunkIndex: 0, + startLine: node.startLine ?? 0, + endLine: node.endLine ?? 0, + }, + ] + : iterateNodeChunks( + node.label, + node.content, + node.filePath, + node.startLine ?? 0, + node.endLine ?? 0, + this.config.chunkSize, + this.config.overlap, + ); + let prevTail = ''; + for await (const chunk of chunks) { + this.options.signal?.throwIfAborted(); + const text = generateEmbeddingText(node, chunk.text, this.config, chunk.chunkIndex, prevTail); + for (const _part of splitEmbeddingText(text, this.config.maxInputChars)) + yield { startLine: chunk.startLine, endLine: chunk.endLine }; + prevTail = this.config.overlap > 0 ? chunk.text.slice(-this.config.overlap) : ''; + } + } + + async readChunks( + receipt: VectorNodeReceipt, + start: number, + length: number, + ): Promise { + if ( + !Number.isSafeInteger(start) || + start < 0 || + !Number.isSafeInteger(length) || + length < 1 || + length > 256 || + start + length > receipt.chunks || + length * (32 * this.options.expectedIdentity.dimensions + 1024 + 4 * receipt.nodeId.length) > + this.options.batchBytes + ) + throw new Error('Invalid legacy vector read batch'); + this.acquire(); + try { + return (await this.rows(receipt, start, length, true)).map((row) => { + if ( + !Array.isArray(row.embedding) || + row.embedding.length !== this.options.expectedIdentity.dimensions || + row.embedding.some((value) => typeof value !== 'number' || !Number.isFinite(value)) + ) + throw new Error('Invalid legacy vector dimensions or values'); + const startLine = count(row.startLine), + endLine = count(row.endLine); + if (endLine < startLine) throw new Error('Invalid legacy vector lines'); + return { + chunkIndex: count(row.chunkIndex), + startLine, + endLine, + embedding: row.embedding as number[], + }; + }); + } finally { + this.active = false; + } + } + async close(): Promise { + if (this.closing) return this.closing; + if (this.active) throw new Error('Cannot close an active legacy vector read'); + this.closed = true; + this.statements.clear(); + this.closing = closeLbugConnection(this.handle, { + strict: true, + databasePath: this.options.path, + }); + try { + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; + } + } +} diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 37afe7489..1a13aa737 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -36,6 +36,15 @@ export interface VectorNodeInput { sourceStartLine?: number; sourceEndLine?: number; } +/** A complete legacy group, outside every interrupted checkpoint window. + * This is a lookup proof only; it is never counted as current paid coverage. */ +export interface LegacyVectorGroup { + nodeId: string; + contentHash: string; + chunks: number; + sourceStartLine: number; + sourceEndLine: number; +} export type VectorWriteChunk = StoredVectorChunk & VectorNodeInput; export interface VectorStoreOptions { directory: string; @@ -572,6 +581,126 @@ export class VectorStore { }); } + /** One bounded metadata pass builds PK lookups beside the pending writer. + * No extra native pool, vector copy, or corpus-sized JS receipt map. A failed + * build leaves complete=false; retry replaces only this auxiliary index. */ + async prepareLegacyReuseIndex( + source: string, + groups: () => AsyncIterable, + excluded: () => AsyncIterable | Iterable, + signal?: AbortSignal, + ): Promise { + if (!/^[a-f0-9]{64}$/.test(source)) throw new Error('Invalid legacy source identity'); + await this.exclusive(async () => { + signal?.throwIfAborted(); + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS LegacyEmbeddingGroup (nodeId STRING, contentHash STRING, chunks INT64, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', + ); + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS LegacyEmbeddingIndex (id STRING, source STRING, complete BOOL, PRIMARY KEY(id))', + ); + const state = await this.query( + "MATCH (s:LegacyEmbeddingIndex {id:'source'}) RETURN s.source AS source, s.complete AS complete", + ); + if (state.length === 1 && state[0].source === source && state[0].complete === true) return; + await this.query( + "MERGE (s:LegacyEmbeddingIndex {id:'source'}) SET s.source=$source, s.complete=false", + { source }, + ); + await this.query('MATCH (g:LegacyEmbeddingGroup) DELETE g'); + let batch: LegacyVectorGroup[] = []; + let bytes = 0; + const flush = async () => { + if (!batch.length) return; + signal?.throwIfAborted(); + await this.query( + 'UNWIND range(1, size($ids)) AS i CREATE (:LegacyEmbeddingGroup {nodeId:$ids[i], contentHash:$hashes[i], chunks:$chunks[i], sourceStartLine:$firstLines[i], sourceEndLine:$lastLines[i]})', + { + ids: batch.map((g) => g.nodeId), + hashes: batch.map((g) => g.contentHash), + chunks: batch.map((g) => g.chunks), + firstLines: batch.map((g) => g.sourceStartLine), + lastLines: batch.map((g) => g.sourceEndLine), + }, + ); + batch = []; + bytes = 0; + }; + for await (const group of groups()) { + signal?.throwIfAborted(); + assertValidGraphNodeId(group.nodeId, 'Legacy vector group'); + const size = 256 + 8 * (group.nodeId.length + group.contentHash.length); + if ( + !group.contentHash || + !nonnegative(group.chunks) || + !group.chunks || + group.chunks > MAX_CHUNKS || + !nonnegative(group.sourceStartLine) || + !nonnegative(group.sourceEndLine) || + group.sourceEndLine < group.sourceStartLine || + size > this.options.batchBytes + ) + throw new Error('Invalid or oversized legacy group receipt'); + if (batch.length === 64 || bytes + size > this.options.batchBytes) await flush(); + batch.push(group); + bytes += size; + } + await flush(); + let unsafe: string[] = []; + const remove = async () => { + if (!unsafe.length) return; + signal?.throwIfAborted(); + await this.query('UNWIND $ids AS id MATCH (g:LegacyEmbeddingGroup {nodeId:id}) DELETE g', { + ids: unsafe, + }); + unsafe = []; + bytes = 0; + }; + for await (const id of excluded()) { + signal?.throwIfAborted(); + assertValidGraphNodeId(id, 'Legacy unsafe window'); + const size = 256 + 8 * id.length; + if (size > this.options.batchBytes) throw new Error('Oversized legacy exclusion'); + if (unsafe.length === 64 || bytes + size > this.options.batchBytes) await remove(); + unsafe.push(id); + bytes += size; + } + await remove(); + signal?.throwIfAborted(); + await this.query("MATCH (s:LegacyEmbeddingIndex {id:'source'}) SET s.complete=true"); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, true); + } + + async legacyReuseGroup(source: string, nodeId: string): Promise { + if (!/^[a-f0-9]{64}$/.test(source)) throw new Error('Invalid legacy source identity'); + assertValidGraphNodeId(nodeId, 'Legacy vector group'); + return this.exclusive(async () => { + const rows = await this.query( + "MATCH (s:LegacyEmbeddingIndex {id:'source'}), (g:LegacyEmbeddingGroup {nodeId:$nodeId}) WHERE s.source=$source AND s.complete=true RETURN g.contentHash AS contentHash, g.chunks AS chunks, g.sourceStartLine AS sourceStartLine, g.sourceEndLine AS sourceEndLine", + { source, nodeId }, + ); + if (!rows.length) return null; + if (rows.length !== 1 || typeof rows[0].contentHash !== 'string' || !rows[0].contentHash) + throw new Error('Corrupt legacy group receipt'); + const group = { + nodeId, + contentHash: rows[0].contentHash, + chunks: count(rows[0].chunks), + sourceStartLine: count(rows[0].sourceStartLine), + sourceEndLine: count(rows[0].sourceEndLine), + }; + if (!group.chunks || group.chunks > MAX_CHUNKS || group.sourceEndLine < group.sourceStartLine) + throw new Error('Corrupt legacy group receipt'); + return group; + }); + } + /** Same immutable job identity: completed groups need no source/model work. */ async completedNodes(ids: readonly string[]): Promise> { if (ids.length > 64 || new Set(ids).size !== ids.length) diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts index 2c6979a44..4a69feee8 100644 --- a/gitnexus/src/storage/embedding-artifact.ts +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -27,6 +27,8 @@ export interface EmbeddingArtifactOwner { revalidateInputs: boolean; /** At most one completed, read-only source for bounded group reuse. */ fallbackArtifact?: string; + /** Alternative sole read-only fallback during first legacy migration. */ + fallbackLegacy?: string; } export interface EmbeddingArtifactRecord { @@ -76,7 +78,10 @@ function ownerRecord(value: unknown): EmbeddingArtifactOwner { !['pending', 'sealed'].includes(owner.phase) || typeof owner.revalidateInputs !== 'boolean' || (owner.fallbackArtifact !== undefined && - (!DIGEST.test(owner.fallbackArtifact) || owner.fallbackArtifact === owner.artifactIdentity)) + (!DIGEST.test(owner.fallbackArtifact) || + owner.fallbackArtifact === owner.artifactIdentity)) || + (owner.fallbackLegacy !== undefined && + (!DIGEST.test(owner.fallbackLegacy) || owner.fallbackArtifact !== undefined)) ) throw new Error('Invalid embedding artifact ownership; artifacts retained'); return { @@ -88,6 +93,7 @@ function ownerRecord(value: unknown): EmbeddingArtifactOwner { phase: owner.phase, revalidateInputs: owner.revalidateInputs, ...(owner.fallbackArtifact === undefined ? {} : { fallbackArtifact: owner.fallbackArtifact }), + ...(owner.fallbackLegacy === undefined ? {} : { fallbackLegacy: owner.fallbackLegacy }), }; } @@ -154,6 +160,7 @@ function validateTransition( a.phase !== 'sealed' || b.phase !== 'pending' || b.fallbackArtifact !== a.artifactIdentity || + b.fallbackLegacy !== undefined || b.artifactIdentity === a.fallbackArtifact ) throw new Error('Pending paid artifacts must be retained until sealed'); @@ -164,7 +171,8 @@ function validateTransition( a.phase !== 'pending' || b.phase !== 'pending' || !b.revalidateInputs || - a.fallbackArtifact !== b.fallbackArtifact + a.fallbackArtifact !== b.fallbackArtifact || + a.fallbackLegacy !== b.fallbackLegacy ) throw new Error( 'Rebased artifact requires current-input validation and its retained fallback', @@ -176,6 +184,7 @@ function validateTransition( a.sourceIdentity !== b.sourceIdentity || a.manifestDigest !== b.manifestDigest || (a.phase === 'sealed' && b.phase !== 'sealed') || + (b.phase === 'pending' && a.fallbackLegacy !== b.fallbackLegacy) || (a.revalidateInputs && !b.revalidateInputs) ) throw new Error('Embedding artifact job identity or sealed state changed'); diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 20de1e8c6..d66fc1aa3 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -5,7 +5,13 @@ import fs from 'node:fs/promises'; import path from 'node:path'; const MAX_RECORD_BYTES = 32 * 1024; -const RECORD_NAMES = new Set(['manifest.json', 'job.json', 'space.json', 'rebase.json']); +const RECORD_NAMES = new Set([ + 'manifest.json', + 'job.json', + 'space.json', + 'rebase.json', + 'legacy.json', +]); const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; export async function embeddingJobDirectory(directory: string): Promise { diff --git a/gitnexus/src/storage/legacy-embedding-source.ts b/gitnexus/src/storage/legacy-embedding-source.ts new file mode 100644 index 000000000..c7be450ab --- /dev/null +++ b/gitnexus/src/storage/legacy-embedding-source.ts @@ -0,0 +1,290 @@ +/** Filesystem-only provenance for one retired, immutable legacy database. + * The caller holds the slot lock and retires/checkpoints the source before + * recording it. No current configuration is used to fill historical fields. */ +import { createHash, randomUUID } from 'node:crypto'; +import { constants, type BigIntStats } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { EmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; +import { assertValidGraphNodeId } from '../core/graph/node-identity.js'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, + syncEmbeddingJobDirectory, +} from './embedding-job.js'; + +const DIGEST = /^[a-f0-9]{64}$/; +const SOURCE = + /^lbug\.generation\.[a-f0-9]{32}$|^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; +const IO_BYTES = 64 * 1024; +const MAX_ID_BYTES = 1024 * 1024; +const hash = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); +const count = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +export interface LegacyEmbeddingSourceRecord { + version: 1; + sourceFile: string; + schemaFingerprint: string; + recordedIdentity?: Partial; + recordedInputIdentity?: string; + /** Descriptor attributes detect replacement/mutation. Ownership and provider + * provenance come from the retired slot and record, never these attributes. */ + file: { dev: string; ino: string; size: string; mtimeNs: string }; + excluded: { digest: string; bytes: number; nodes: number }; +} +export interface ResolvedLegacyEmbeddingSource extends LegacyEmbeddingSourceRecord { + identity: string; + path: string; + excludedIds: () => AsyncIterable; +} + +function identityFields( + value: Partial | undefined, +): Partial | undefined { + if (value === undefined) return undefined; + if ( + !value || + typeof value !== 'object' || + (value.provider !== undefined && !/^(local|http:[a-f0-9]{64})$/.test(value.provider)) || + (value.model !== undefined && + (typeof value.model !== 'string' || !value.model || value.model.length > 2048)) || + (value.dimensions !== undefined && (!count(value.dimensions) || !value.dimensions)) + ) + throw new Error('Invalid recorded legacy embedding identity'); + return { + ...(value.provider === undefined ? {} : { provider: value.provider }), + ...(value.model === undefined ? {} : { model: value.model }), + ...(value.dimensions === undefined ? {} : { dimensions: value.dimensions }), + }; +} +function record(value: unknown): LegacyEmbeddingSourceRecord { + const r = value as LegacyEmbeddingSourceRecord; + if ( + !r || + r.version !== 1 || + typeof r.sourceFile !== 'string' || + !SOURCE.test(r.sourceFile) || + typeof r.schemaFingerprint !== 'string' || + !r.schemaFingerprint || + r.schemaFingerprint.length > 2048 || + (r.recordedInputIdentity !== undefined && !DIGEST.test(r.recordedInputIdentity)) || + !r.file || + ![r.file.dev, r.file.ino, r.file.size, r.file.mtimeNs].every( + (v) => typeof v === 'string' && /^\d+$/.test(v), + ) || + !r.excluded || + !DIGEST.test(r.excluded.digest) || + !count(r.excluded.bytes) || + !count(r.excluded.nodes) + ) + throw new Error('Invalid legacy source receipt; source retained'); + const recordedIdentity = identityFields(r.recordedIdentity); + return { + version: 1, + sourceFile: r.sourceFile, + schemaFingerprint: r.schemaFingerprint, + ...(recordedIdentity === undefined ? {} : { recordedIdentity }), + ...(r.recordedInputIdentity === undefined + ? {} + : { recordedInputIdentity: r.recordedInputIdentity }), + file: { dev: r.file.dev, ino: r.file.ino, size: r.file.size, mtimeNs: r.file.mtimeNs }, + excluded: { digest: r.excluded.digest, bytes: r.excluded.bytes, nodes: r.excluded.nodes }, + }; +} +function fileIdentity(stat: BigIntStats): LegacyEmbeddingSourceRecord['file'] { + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy source must be an owned regular file'); + return { + dev: String(stat.dev), + ino: String(stat.ino), + size: String(stat.size), + mtimeNs: String(stat.mtimeNs), + }; +} +async function sourceFile(root: string, r: LegacyEmbeddingSourceRecord): Promise { + const file = path.join(root, r.sourceFile); + if ( + JSON.stringify(fileIdentity(await fs.lstat(file, { bigint: true }))) !== JSON.stringify(r.file) + ) + throw new Error('Legacy source changed after retirement; artifacts retained'); + // A read-only migration cannot recover an interrupted native write. Its + // graph-phase owner must checkpoint/recover it before writing this receipt. + for (const suffix of ['.wal', '.shadow']) { + try { + const stat = await fs.lstat(file + suffix); + if (!stat.isFile() || stat.size) + throw new Error('Legacy source needs native recovery before reuse'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + return file; +} +async function legacyRoot(root: string, create: boolean): Promise { + let directory = await embeddingJobDirectory(root); + for (const part of ['embeddings', 'legacy']) { + const next = path.join(directory, part); + if (create) { + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + +/** Streams checkpoint exclusions once. The small receipt is published last, + * after the spool is durable. The native source is never copied or modified. */ +export async function recordLegacyEmbeddingSource( + root: string, + input: Pick< + LegacyEmbeddingSourceRecord, + 'sourceFile' | 'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity' + >, + excluded: AsyncIterable | Iterable, + signal?: AbortSignal, +): Promise { + root = await embeddingJobDirectory(root); + if (!SOURCE.test(input.sourceFile)) + throw new Error('Legacy source must be a retired owned graph'); + const file = fileIdentity(await fs.lstat(path.join(root, input.sourceFile), { bigint: true })); + const parent = await legacyRoot(root, true); + const temporary = path.join(parent, `.prepare-${randomUUID()}`); + await fs.mkdir(temporary, { mode: 0o700 }); + try { + const handle = await fs.open(path.join(temporary, 'excluded.jsonl'), 'wx', 0o600); + const checksum = createHash('sha256'); + let bytes = 0, + nodes = 0, + pending = '', + pendingBytes = 0; + try { + for await (const id of excluded) { + signal?.throwIfAborted(); + assertValidGraphNodeId(id, 'Legacy checkpoint'); + const line = JSON.stringify(id) + '\n'; + const length = Buffer.byteLength(line); + if (length > MAX_ID_BYTES || !Number.isSafeInteger(bytes + length)) + throw new Error('Legacy exclusion exceeds its bounded size'); + checksum.update(line); + bytes += length; + nodes++; + pending += line; + pendingBytes += length; + if (pendingBytes >= IO_BYTES) { + await handle.writeFile(pending); + pending = ''; + pendingBytes = 0; + } + } + if (pendingBytes) await handle.writeFile(pending); + await handle.sync(); + } finally { + await handle.close(); + } + signal?.throwIfAborted(); + const receipt = record({ + version: 1, + ...input, + file, + excluded: { digest: checksum.digest('hex'), bytes, nodes }, + }); + await sourceFile(root, receipt); + const identity = hash(receipt); + await writeEmbeddingJobRecord(temporary, 'legacy.json', receipt); + const destination = path.join(parent, identity); + try { + await fs.rename(temporary, destination); + } catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) + throw error; + // An identical request is idempotent, but a corrupt prior spool is not + // silently replaced. Fully validate it before trusting the same name. + const prior = await readLegacyEmbeddingSource(root, identity); + for await (const _id of prior.excludedIds()) signal?.throwIfAborted(); + } + await syncEmbeddingJobDirectory(parent); + return identity; + } finally { + await fs.rm(temporary, { recursive: true, force: true }); + } +} + +async function* exclusions( + directory: string, + expected: LegacyEmbeddingSourceRecord['excluded'], +): AsyncGenerator { + const file = path.join(directory, 'excluded.jsonl'); + if (!(await fs.lstat(file)).isFile()) throw new Error('Legacy exclusions must be a regular file'); + const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size !== expected.bytes) + throw new Error('Legacy exclusions size mismatch'); + const buffer = Buffer.alloc(IO_BYTES), + checksum = createHash('sha256'); + let pending = Buffer.alloc(0), + bytes = 0, + nodes = 0; + for (;;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, bytes); + if (!bytesRead) break; + bytes += bytesRead; + checksum.update(buffer.subarray(0, bytesRead)); + const data = pending.length + ? Buffer.concat([pending, buffer.subarray(0, bytesRead)]) + : buffer.subarray(0, bytesRead); + let offset = 0; + for (;;) { + const end = data.indexOf(10, offset); + if (end < 0) break; + if (end - offset + 1 > MAX_ID_BYTES) + throw new Error('Legacy exclusion exceeds its bounded size'); + const id: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)), + ); + assertValidGraphNodeId(id, 'Legacy checkpoint'); + nodes++; + yield id; + offset = end + 1; + } + pending = Buffer.from(data.subarray(offset)); + if (pending.length >= MAX_ID_BYTES || bytes > expected.bytes) + throw new Error('Legacy exclusions exceed their bounds'); + } + if ( + pending.length || + bytes !== expected.bytes || + nodes !== expected.nodes || + checksum.digest('hex') !== expected.digest + ) + throw new Error('Legacy exclusions checksum mismatch; source retained'); + } finally { + await handle.close(); + } +} + +export async function readLegacyEmbeddingSource( + root: string, + identity: string, +): Promise { + if (!DIGEST.test(identity)) throw new Error('Invalid legacy source identity'); + root = await embeddingJobDirectory(root); + const directory = await embeddingJobDirectory(path.join(await legacyRoot(root, false), identity)); + const receipt = record(await readEmbeddingJobRecord(directory, 'legacy.json')); + if (hash(receipt) !== identity) + throw new Error('Legacy source identity mismatch; source retained'); + return { + ...receipt, + identity, + path: await sourceFile(root, receipt), + excludedIds: () => exclusions(directory, receipt.excluded), + }; +} diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 3ebc18e25..5f4a3ef84 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -136,6 +136,9 @@ export interface RepoMeta { artifactIdentity?: string; spaceKey?: string; }; + /** Direct receipt for a retained legacy source; missing historical fields + * remain unknown. The source is immutable and separate from graphPath. */ + legacyVectorSource?: string; /** Published only after complete coverage, native durability and index validation. */ vectorGeneration?: { version: 1; diff --git a/gitnexus/test/integration/legacy-embedding-controller.test.ts b/gitnexus/test/integration/legacy-embedding-controller.test.ts new file mode 100644 index 000000000..cb83b87fb --- /dev/null +++ b/gitnexus/test/integration/legacy-embedding-controller.test.ts @@ -0,0 +1,305 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + runPublishedEmbeddingJob, + type PublishedEmbeddingJobOptions, +} from '../../src/core/embeddings/embedding-job.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { LegacyVectorSource } from '../../src/core/lbug/legacy-vector-source.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { recordLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { readEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; + +const roots: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); +async function fixture(count = 130, unsafe: string[] = [], knownProvider = true) { + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-controller-')); + roots.push(root); + const identity = { provider: 'local', model: 'test', dimensions: 4 }; + const config = resolveEmbeddingConfig({ threads: 1, subBatchSize: 8 }); + const sourceFile = 'lbug.generation.' + 'a'.repeat(32); + const sourcePath = path.join(root, sourceFile); + const id = (i: number) => `Const:a.ts:n${String(i).padStart(5, '0')}`; + const node = (key: string): EmbeddableNode => ({ + id: key, + name: key.split(':').at(-1) ?? key, + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, + }); + const handle = await openLbugConnection(lbug, sourcePath, { + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + }); + const close = (result: lbug.QueryResult | lbug.QueryResult[]) => { + for (const r of Array.isArray(result) ? result : [result]) r.close(); + }; + try { + close( + await handle.conn.query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ), + ); + const insert = await handle.conn.prepare( + 'CREATE (:CodeEmbedding {id:$id,nodeId:$nodeId,chunkIndex:0,startLine:1,endLine:1,embedding:$embedding,contentHash:$hash})', + ); + for (let i = 0; i < count; i++) + close( + await handle.conn.execute(insert, { + id: id(i) + ':0', + nodeId: id(i), + embedding: [1, 0, 0, 0], + hash: contentHashForNode(node(id(i)), config), + }), + ); + close(await handle.conn.query('CHECKPOINT')); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: sourcePath }); + } + const legacyIdentity = await recordLegacyEmbeddingSource( + root, + { + sourceFile, + schemaFingerprint: 'schema', + recordedIdentity: knownProvider ? identity : { dimensions: 4 }, + }, + unsafe, + ); + let generation = 1; + const graphFile = () => 'lbug.generation.' + String(generation).padStart(32, '0'); + const graphGeneration = () => ({ + version: 1 as const, + id: String(generation).padStart(32, '0'), + coveredContent: 'b'.repeat(64), + }); + const meta: RepoMeta = { + repoPath: root, + lastCommit: 'commit', + indexedAt: '2026-11-04T00:00:00Z', + graphPath: graphFile(), + graphGeneration: graphGeneration(), + legacyVectorSource: legacyIdentity, + schemaFingerprint: 'schema', + parseCacheIdentity: { base: 'parser' }, + scopeExtractionReceipt: 1, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'test', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: '127', + libc: 'glibc', + }, + invokedArtifact: { path: 'test', digest: 'test' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'build', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'runtime', + }, + }, + }; + await fs.writeFile(path.join(root, graphFile()), 'graph opened by the deterministic test source'); + await saveMeta(root, meta); + const changed = new Map(); + const events: string[] = []; + const provider = vi.fn(async (texts: string[]) => { + events.push('provider'); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }); + const initializeProvider = vi.fn(async () => {}); + const source = { + scan: async function* (label: string) { + if (label === 'Const') + for (let i = 0; i < count; i++) + yield { + id: id(i), + contentBytes: Buffer.byteLength(changed.get(id(i)) ?? node(id(i)).content), + metadataBytes: 1024, + }; + }, + fetch: async (_label: string, ids: readonly string[]) => + ids.map((key) => ({ ...node(key), content: changed.get(key) ?? node(key).content })), + close: async () => {}, + }; + const options = (): PublishedEmbeddingJobOptions => ({ + metadataDirectory: root, + graphPath: path.join(root, graphFile()), + config, + resources: { + graphPoolBytes: 16 * 1024 * 1024, + vectorPoolBytes: 64 * 1024 * 1024, + queueBytes: 16 * 1024 * 1024, + nativeThreads: 1, + modelBytes: 0, + modelHeapMb: 0, + reservedBytes: 96 * 1024 * 1024, + }, + metadata: { + loadMetadata: () => loadMeta(root), + saveMetadata: (value) => saveMeta(root, value), + }, + log: (message) => events.push(message), + }); + const run = (extra: Partial = {}) => + runPublishedEmbeddingJob( + { ...options(), ...extra }, + { identity: () => identity, openSource: async () => source, provider, initializeProvider }, + ); + const nextGraph = async () => { + const previous = await loadMeta(root); + if (!previous) throw new Error('missing metadata'); + generation++; + await fs.writeFile(path.join(root, graphFile()), 'new graph'); + await saveMeta(root, { + ...previous, + graphPath: graphFile(), + graphGeneration: graphGeneration(), + legacyVectorSource: undefined, + }); + }; + return { + root, + run, + options, + nextGraph, + provider, + initializeProvider, + changed, + events, + id, + sourcePath, + legacyIdentity, + }; +} + +describe('legacy migration through the published job controller', () => { + it('keeps provider batches full when legacy groups are ineligible for reuse', async () => { + const f = await fixture( + 16, + Array.from({ length: 16 }, (_, i) => `Const:a.ts:n${String(i).padStart(5, '0')}`), + ); + expect((await f.run()).phase).toBe('ready'); + expect(f.provider.mock.calls.map(([texts]) => texts.length)).toEqual([8, 8]); + }); + it('retains one legacy source and pending store across two crashes and graph changes with zero paid replay', async () => { + const f = await fixture(); + const before = await fs.stat(f.sourcePath); + let scans = 0; + const prepare = VectorStore.prototype.prepareLegacyReuseIndex; + vi.spyOn(VectorStore.prototype, 'prepareLegacyReuseIndex').mockImplementation(function ( + this: VectorStore, + source, + groups, + excluded, + signal, + ) { + return prepare.call( + this, + source, + async function* () { + scans++; + yield* groups(); + }, + excluded, + signal, + ); + }); + await expect( + f.run({ + onProgress: (state) => { + if (state.reusedNodes === 64) throw new Error('first stop'); + }, + }), + ).rejects.toThrow('first stop'); + const first = (await loadMeta(f.root))?.embeddingJob; + expect(first?.artifactIdentity).toBeDefined(); + await f.nextGraph(); + await expect( + f.run({ + onProgress: (state) => { + if (state.reusedNodes === 128) throw new Error('second stop'); + }, + }), + ).rejects.toThrow('second stop'); + expect((await loadMeta(f.root))?.embeddingJob?.artifactIdentity).toBe(first?.artifactIdentity); + await f.nextGraph(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(130); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(scans).toBe(1); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); + expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); + const owner = (await readEmbeddingJobRecord( + path.join(f.root, 'embeddings', 'spaces', first?.spaceKey ?? ''), + 'space.json', + )) as { owner: { fallbackLegacy?: string } }; + expect(owner.owner.fallbackLegacy).toBeUndefined(); + }); + it('regenerates only changed inputs and groups in the unsafe checkpoint window', async () => { + const f = await fixture(5, ['Const:a.ts:n00002']); + f.changed.set(f.id(1), 'const changed = 2;'); + const result = await f.run(); + expect(result.state?.reusedNodes).toBe(3); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(2); + expect(f.events[0]).toContain('chunk size/overlap'); + expect(f.events.indexOf('provider')).toBeGreaterThan(0); + }); + it('reports unknown provider provenance before requests and retains its source without opening it', async () => { + const f = await fixture(3, [], false); + const open = vi.spyOn(LegacyVectorSource, 'open'); + expect((await f.run()).phase).toBe('ready'); + expect(open).not.toHaveBeenCalled(); + expect(f.events[0]).toContain('provider/model/dimensions provenance is unknown'); + expect(f.events.indexOf('provider')).toBeGreaterThan(0); + expect((await fs.stat(f.sourcePath)).isFile()).toBe(true); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + }); + it('rejects changed receipts before model initialization or requests', async () => { + const f = await fixture(3); + await fs.writeFile( + path.join(f.root, 'embeddings', 'legacy', f.legacyIdentity, 'legacy.json'), + '{}', + ); + await expect(f.run()).rejects.toThrow('Malformed'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + }); + it('rejects a hard-linked live graph as a legacy source before paid work', async () => { + const f = await fixture(3); + const graph = f.options().graphPath; + await fs.unlink(graph); + await fs.link(f.sourcePath, graph); + await expect(f.run()).rejects.toThrow('distinct from the mutable graph'); + expect(f.provider).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/integration/legacy-vector-reuse-index.test.ts b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts new file mode 100644 index 000000000..35b0f10b3 --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts @@ -0,0 +1,135 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type LegacyVectorGroup } from '../../src/core/lbug/vector-store.js'; + +const source = 'a'.repeat(64); +const roots: string[] = []; +const handles: VectorStore[] = []; +async function fixture() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-index-')); + roots.push(directory); + const options = { + directory, + identity: 'b'.repeat(64), + dimensions: 4, + bufferPoolBytes: 8 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + }; + const store = await VectorStore.open({ ...options, mode: 'create' }); + handles.push(store); + return { store, options }; +} +const group = (n: number): LegacyVectorGroup => ({ + nodeId: `Function:a.ts:n${n}`, + contentHash: `hash-${n}`, + chunks: 2, + sourceStartLine: n * 3, + sourceEndLine: n * 3 + 2, +}); +async function* groups(n: number) { + for (let i = 0; i < n; i++) yield group(i); +} +afterEach(async () => { + for (const handle of handles.splice(0)) await handle.close(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('bounded native legacy receipt index', () => { + it('streams receipt batches, excludes unsafe groups, and resumes without rescanning', async () => { + const { store, options } = await fixture(); + const scan = vi.fn(() => groups(1000)); + const excluded = vi.fn(() => [group(2).nodeId, group(999).nodeId]); + await store.prepareLegacyReuseIndex(source, scan, excluded); + expect(await store.legacyReuseGroup(source, group(998).nodeId)).toEqual(group(998)); + expect(await store.legacyReuseGroup(source, group(2).nodeId)).toBeNull(); + expect(await store.legacyReuseGroup('c'.repeat(64), group(998).nodeId)).toBeNull(); + expect(await store.completedNodes([group(998).nodeId])).toEqual(new Map()); + await store.close(); + const reopened = await VectorStore.open({ ...options, mode: 'resume' }); + handles.push(reopened); + await reopened.prepareLegacyReuseIndex(source, scan, excluded); + expect(scan).toHaveBeenCalledOnce(); + expect(excluded).toHaveBeenCalledOnce(); + expect(await reopened.legacyReuseGroup(source, group(998).nodeId)).toEqual(group(998)); + }); + it('rebuilds an interrupted receipt pass without deleting completed paid vectors', async () => { + const { store } = await fixture(); + const id = 'Function:a.ts:paid'; + await store.beginNode(id, 'paid-hash'); + await store.append(id, 'paid-hash', [ + { chunkIndex: 0, startLine: 1, endLine: 2, embedding: [1, 0, 0, 0] }, + ]); + await store.finishNode(id, 'paid-hash', 1); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield* groups(65); + throw new Error('interrupted'); + }, + () => [], + ), + ).rejects.toThrow('interrupted'); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await store.prepareLegacyReuseIndex( + source, + () => groups(3), + () => [], + ); + expect(await store.legacyReuseGroup(source, group(2).nodeId)).toEqual(group(2)); + expect(await store.legacyReuseGroup(source, group(64).nodeId)).toBeNull(); + expect((await store.completedNodes([id])).size).toBe(1); + }); + it('rejects duplicate or oversized receipts before marking the source complete', async () => { + const { store } = await fixture(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield group(0); + yield group(0); + }, + () => [], + ), + ).rejects.toThrow(); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield { ...group(1), contentHash: 'x'.repeat(1024 * 1024) }; + }, + () => [], + ), + ).rejects.toThrow('oversized'); + expect(await store.legacyReuseGroup(source, group(1).nodeId)).toBeNull(); + }); + it('keeps cancellation incomplete and refuses writes to a published artifact', async () => { + const { store } = await fixture(); + const controller = new AbortController(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield* groups(65); + controller.abort(); + yield group(66); + }, + () => [], + controller.signal, + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await store.seal(0, 0); + await expect( + store.prepareLegacyReuseIndex( + source, + () => groups(1), + () => [], + ), + ).rejects.toThrow(); + }); +}); diff --git a/gitnexus/test/integration/legacy-vector-source.test.ts b/gitnexus/test/integration/legacy-vector-source.test.ts new file mode 100644 index 000000000..38ed546cf --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-source.test.ts @@ -0,0 +1,264 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type StoredVectorChunk } from '../../src/core/lbug/vector-store.js'; +import { + LegacyVectorSource, + legacyVectorCompatibility, + type LegacyVectorSourceOptions, +} from '../../src/core/lbug/legacy-vector-source.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { chunkNode } from '../../src/core/embeddings/chunker.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { + generateEmbeddingText, + splitEmbeddingText, +} from '../../src/core/embeddings/text-generator.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import { isShortLabel, type EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { createEmbeddingWorkManifest } from '../../src/core/embeddings/embedding-work-manifest.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { runEmbeddingJob } from '../../src/core/embeddings/embedding-job-runner.js'; + +const roots: string[] = []; +const handles: Array<{ close(): Promise }> = []; +const identity = { provider: 'local', model: 'recorded', dimensions: 4 }; +const config = resolveEmbeddingConfig({ chunkSize: 200, overlap: 20, maxInputChars: 900 }); +const node = (kind: 'File' | 'Const', content = 'const n = 1;'): EmbeddableNode => ({ + id: kind === 'File' ? 'File:a.txt' : 'Const:a.txt:n', + name: 'n', + label: kind, + filePath: 'a.txt', + content, + startLine: 1, + endLine: content.split('\n').length, +}); +async function legacyChunks(input: EmbeddableNode): Promise { + const chunks = isShortLabel(input.label) + ? [ + { + text: input.content, + chunkIndex: 0, + startLine: input.startLine ?? 0, + endLine: input.endLine ?? 0, + }, + ] + : await chunkNode( + input.label, + input.content, + input.filePath, + input.startLine ?? 0, + input.endLine ?? 0, + config.chunkSize, + config.overlap, + ); + const rows: StoredVectorChunk[] = []; + let tail = ''; + for (const chunk of chunks) { + const text = generateEmbeddingText(input, chunk.text, config, chunk.chunkIndex, tail); + for (const _part of splitEmbeddingText(text, config.maxInputChars)) + rows.push({ + chunkIndex: rows.length, + startLine: chunk.startLine, + endLine: chunk.endLine, + embedding: [1, 0, 0, 0], + }); + tail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + } + return rows; +} +async function fixture( + input: EmbeddableNode, + adjust?: (rows: StoredVectorChunk[]) => StoredVectorChunk[], +) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-source-')); + roots.push(root); + const dbPath = path.join(root, 'lbug'); + const handle = await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: 8 * 1024 * 1024, + threads: 1, + }); + const close = (result: lbug.QueryResult | lbug.QueryResult[]) => { + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + const hash = contentHashForNode(input, config); + const chunks = await legacyChunks(input); + try { + close( + await handle.conn.query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ), + ); + const insert = await handle.conn.prepare( + 'CREATE (:CodeEmbedding {id:$id,nodeId:$nodeId,chunkIndex:$chunkIndex,startLine:$startLine,endLine:$endLine,embedding:$embedding,contentHash:$contentHash})', + ); + for (const chunk of adjust ? adjust(chunks) : chunks) + close( + await handle.conn.execute(insert, { + id: `${input.id}:${chunk.chunkIndex}`, + nodeId: input.id, + contentHash: hash, + ...chunk, + }), + ); + close(await handle.conn.query('CHECKPOINT')); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: dbPath }); + } + const directory = path.join(root, 'pending'); + await fs.mkdir(directory); + const store = await VectorStore.open({ + directory, + identity: 'b'.repeat(64), + dimensions: 4, + bufferPoolBytes: 32 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + mode: 'create', + }); + handles.push(store); + const options: LegacyVectorSourceOptions = { + path: dbPath, + sourceIdentity: 'a'.repeat(64), + recordedIdentity: identity, + expectedIdentity: identity, + recordedInputIdentity: 'c'.repeat(64), + expectedInputIdentity: 'c'.repeat(64), + config, + bufferPoolBytes: 8 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + }; + const open = async ( + override: Partial = {}, + excluded: string[] = [], + ) => { + const source = await LegacyVectorSource.open( + { ...options, ...override }, + store, + () => excluded, + ); + handles.push(source); + return source; + }; + return { root, directory, store, options, open, hash, chunks }; +} +afterEach(async () => { + for (const handle of handles.splice(0).reverse()) await handle.close(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('verified legacy vector reuse', () => { + it('relocates chunk spans without treating the last chunk as the whole node range', async () => { + const original = { + ...node('File', Array.from({ length: 80 }, (_, i) => `const v${i} = ${i};`).join('\n')), + endLine: 200, + }; + const f = await fixture(original); + expect(Math.max(...f.chunks.map((chunk) => chunk.endLine))).toBeLessThan(original.endLine); + const reuse = await f.open(); + const moved = { ...original, startLine: 11, endLine: 210 }; + expect(await reuse.reusable(moved.id, f.hash, moved)).toMatchObject({ + complete: true, + sourceStartLine: 1, + sourceEndLine: undefined, + }); + }); + it('rejects missing or different provider provenance before touching the database', async () => { + expect(legacyVectorCompatibility(undefined, identity)).toContain('unknown'); + expect( + legacyVectorCompatibility({ ...identity, provider: 'http:' + 'a'.repeat(64) }, identity), + ).toContain('differ'); + expect(legacyVectorCompatibility(identity, identity)).toBeUndefined(); + const f = await fixture(node('Const')); + await expect(f.open({ path: '/missing', recordedIdentity: undefined })).rejects.toThrow( + 'provenance is unknown', + ); + await expect( + f.open({ path: '/missing', recordedInputIdentity: 'd'.repeat(64) }), + ).rejects.toThrow('input provenance differs'); + }); + it('validates a whole multi-chunk group, copies bounded batches, and relocates with zero provider calls', async () => { + const original = node( + 'File', + Array.from({ length: 80 }, (_, i) => `export const value${i} = ${i};`).join('\n'), + ); + const f = await fixture(original); + expect(f.chunks.length).toBeGreaterThan(1); + const reuse = await f.open(); + const moved = { ...original, startLine: 11, endLine: (original.endLine ?? 0) + 10 }; + const source = { + scan: async function* (label: string) { + if (label === 'File') + yield { + id: moved.id, + contentBytes: Buffer.byteLength(moved.content), + metadataBytes: 512, + }; + }, + fetch: async () => [moved], + }; + const manifest = await createEmbeddingWorkManifest(f.directory, 'e'.repeat(64), source); + const journal = await EmbeddingJobJournal.create(f.directory, 'b'.repeat(64), manifest); + const provider = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([1, 0, 0, 0])), + ); + const result = await runEmbeddingJob({ + directory: f.directory, + manifest, + journal, + source, + store: f.store, + reuse, + provider, + config, + queueBytes: 8 * 1024 * 1024, + }); + expect(provider).not.toHaveBeenCalled(); + expect(result.state.reusedNodes).toBe(1); + const receipt = await f.store.reusable(moved.id, f.hash); + if (!receipt) throw new Error('Missing migrated receipt'); + expect(receipt.chunks).toBe(f.chunks.length); + expect(await f.store.readChunks(receipt, 0, Math.min(8, receipt.chunks))).toEqual( + f.chunks.slice(0, 8).map((chunk) => ({ + ...chunk, + startLine: chunk.startLine + 10, + endLine: chunk.endLine + 10, + })), + ); + }); + it('does not infer unknown multi-chunk settings from identical line spans', async () => { + const input = node('File', 'a'.repeat(800)); + const f = await fixture(input); + expect(f.chunks.length).toBeGreaterThan(1); + expect(new Set(f.chunks.map((c) => `${c.startLine}:${c.endLine}`)).size).toBe(1); + const log = vi.fn(); + const reuse = await f.open({ recordedInputIdentity: undefined, log }); + expect(await reuse.reusable(input.id, f.hash, input)).toBeNull(); + expect(log).toHaveBeenCalledOnce(); + expect(log.mock.calls[0][0]).toContain('chunk size/overlap'); + }); + it('can prove an unchunked short-label input without inventing a historical input receipt', async () => { + const input = node('Const'); + const f = await fixture(input); + const reuse = await f.open({ recordedInputIdentity: undefined }); + expect(await reuse.reusable(input.id, f.hash, input)).toMatchObject({ + chunks: f.chunks.length, + }); + expect(reuse.options.recordedInputIdentity).toBeUndefined(); + }); + it('rejects changed hashes, holes, and groups in an unsafe checkpoint window', async () => { + const input = node('File', 'x'.repeat(1000)); + const f = await fixture(input, (rows) => rows.filter((row) => row.chunkIndex !== 1)); + expect(await (await f.open()).reusable(input.id, f.hash, input)).toBeNull(); + const intact = await fixture(input); + const excluded = await intact.open({}, [input.id]); + expect(await excluded.reusable(input.id, intact.hash, input)).toBeNull(); + const changed = await fixture(node('Const')); + expect( + await (await changed.open()).reusable('Const:a.txt:n', 'different', node('Const')), + ).toBeNull(); + }); +}); diff --git a/gitnexus/test/unit/legacy-embedding-source.test.ts b/gitnexus/test/unit/legacy-embedding-source.test.ts new file mode 100644 index 000000000..cf3599816 --- /dev/null +++ b/gitnexus/test/unit/legacy-embedding-source.test.ts @@ -0,0 +1,197 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + recordLegacyEmbeddingSource, + readLegacyEmbeddingSource, +} from '../../src/storage/legacy-embedding-source.js'; +import { + adoptEmbeddingArtifact, + readEmbeddingArtifactSpace, +} from '../../src/storage/embedding-artifact.js'; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-receipt-')); + roots.push(root); + const sourceFile = 'lbug.generation.' + 'a'.repeat(32); + await fs.writeFile(path.join(root, sourceFile), 'retired database'); + const input = { + sourceFile, + schemaFingerprint: 'schema', + recordedIdentity: { provider: 'local', model: 'old', dimensions: 4 }, + }; + return { root, input, source: path.join(root, sourceFile) }; +} +const collect = async (ids: AsyncIterable) => { + const result: string[] = []; + for await (const id of ids) result.push(id); + return result; +}; + +describe('retained legacy source receipts', () => { + it('retains the sole legacy fallback through rebases and drops it only after sealing', async () => { + const f = await fixture(); + const legacy = await recordLegacyEmbeddingSource(f.root, f.input, []); + const space = { + vectorIdentity: 'a'.repeat(64), + inputIdentity: 'b'.repeat(64), + dimensions: 4, + schema: 'vectors', + }; + const owner = { + artifactIdentity: 'c'.repeat(64), + jobIdentity: 'c'.repeat(64), + sourceIdentity: 'd'.repeat(64), + graphGeneration: 'e'.repeat(32), + manifestDigest: 'f'.repeat(64), + phase: 'pending' as const, + revalidateInputs: false, + fallbackLegacy: legacy, + }; + const validate = async () => {}; + const first = await adoptEmbeddingArtifact(f.root, space, null, owner, validate); + const rebased = { + ...owner, + jobIdentity: '1'.repeat(64), + graphGeneration: '2'.repeat(32), + revalidateInputs: true, + }; + await expect( + adoptEmbeddingArtifact( + f.root, + space, + first, + { ...rebased, fallbackLegacy: undefined }, + validate, + ), + ).rejects.toThrow('retained fallback'); + await expect( + adoptEmbeddingArtifact( + f.root, + space, + first, + { ...owner, fallbackArtifact: '3'.repeat(64) }, + validate, + ), + ).rejects.toThrow('ownership'); + const next = await adoptEmbeddingArtifact(f.root, space, first, rebased, validate); + expect((await readEmbeddingArtifactSpace(f.root, space, validate))?.owner.fallbackLegacy).toBe( + legacy, + ); + await expect( + adoptEmbeddingArtifact( + f.root, + space, + next, + { ...rebased, fallbackLegacy: undefined }, + validate, + ), + ).rejects.toThrow('changed'); + const sealed = await adoptEmbeddingArtifact( + f.root, + space, + next, + { ...rebased, phase: 'sealed', fallbackLegacy: undefined }, + validate, + ); + expect(sealed.owner.fallbackLegacy).toBeUndefined(); + }); + it('streams a large exclusion list and is idempotent without copying the native source', async () => { + const f = await fixture(); + const before = await fs.stat(f.source); + let yielded = 0; + const ids = async function* () { + for (let i = 0; i < 12000; i++) { + yielded++; + yield `Const:a.ts:n${i}`; + } + }; + const identity = await recordLegacyEmbeddingSource(f.root, f.input, ids()); + expect(yielded).toBe(12000); + const retained = await readLegacyEmbeddingSource(f.root, identity); + expect(retained.recordedIdentity).toEqual(f.input.recordedIdentity); + expect(retained.recordedInputIdentity).toBeUndefined(); + expect((await collect(retained.excludedIds())).length).toBe(12000); + expect(await recordLegacyEmbeddingSource(f.root, f.input, ids())).toBe(identity); + expect((await fs.stat(f.source)).ino).toBe(before.ino); + expect((await fs.stat(f.source)).mtimeMs).toBe(before.mtimeMs); + expect( + (await fs.stat(path.join(f.root, 'embeddings', 'legacy', identity, 'legacy.json'))).size, + ).toBeLessThan(2048); + }); + it('preserves unknown fields and refuses paths outside a retired graph', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource( + f.root, + { ...f.input, recordedIdentity: { dimensions: 4 } }, + [], + ); + expect((await readLegacyEmbeddingSource(f.root, identity)).recordedIdentity).toEqual({ + dimensions: 4, + }); + await expect( + recordLegacyEmbeddingSource(f.root, { ...f.input, sourceFile: '../lbug' }, []), + ).rejects.toThrow('retired'); + await expect( + recordLegacyEmbeddingSource(f.root, { ...f.input, sourceFile: 'lbug' }, []), + ).rejects.toThrow('retired'); + }); + it('rejects replaced sources, source symlinks, and unrecovered WALs', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + await fs.rename(f.source, f.source + '.old'); + await fs.writeFile(f.source, 'retired database'); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow( + 'changed after retirement', + ); + await fs.unlink(f.source); + await fs.symlink(f.source + '.old', f.source); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('regular file'); + await fs.unlink(f.source); + await fs.rename(f.source + '.old', f.source); + await fs.writeFile(f.source + '.wal', 'not checkpointed'); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('native recovery'); + }); + it('validates the complete exclusion checksum before a native index may become complete', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x']); + const file = path.join(f.root, 'embeddings', 'legacy', identity, 'excluded.jsonl'); + await fs.writeFile(file, '"Const:a.ts:y"\n'); + const retained = await readLegacyEmbeddingSource(f.root, identity); + await expect(collect(retained.excludedIds())).rejects.toThrow('checksum mismatch'); + await expect(recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x'])).rejects.toThrow( + 'checksum mismatch', + ); + }); + it('rejects a corrupt bounded receipt instead of selecting another source', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + await fs.writeFile(path.join(f.root, 'embeddings', 'legacy', identity, 'legacy.json'), '{}'); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('Malformed'); + }); + it('cleans only its unpublished spool on cancellation and retains source data', async () => { + const f = await fixture(); + const signal = AbortSignal.abort(); + await expect( + recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x'], signal), + ).rejects.toThrow(); + expect(await fs.readFile(f.source, 'utf8')).toBe('retired database'); + expect(await fs.readdir(path.join(f.root, 'embeddings', 'legacy'))).toEqual([]); + }); + it('bounds corrupt exclusion framing and rejects directory symlinks', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + const directory = path.join(f.root, 'embeddings', 'legacy', identity); + await fs.rename(directory, directory + '.original'); + await fs.symlink(directory + '.original', directory); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('owned directory'); + await expect( + recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:' + 'x'.repeat(1024 * 1024)]), + ).rejects.toThrow('bounded size'); + }); +}); From ff215e0b5315cdd912b8f56a99b5d28b00c02eb8 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 11:20:26 +0000 Subject: [PATCH 22/67] feat(analyze): account for supervised phases Charge parent memory once per phase. Carry allowlisted progress through worker IPC and preserve counters across logs without completing the job. Refs #3598 --- gitnexus/src/core/analysis-progress.ts | 62 +++++++++++++++++++ gitnexus/src/core/analysis-resource-budget.ts | 23 +++++-- gitnexus/src/server/analyze-launch.ts | 9 ++- gitnexus/src/server/analyze-worker-core.ts | 11 +++- .../src/server/analyze-worker-protocol.ts | 3 +- gitnexus/src/server/ops-snapshot.ts | 59 +----------------- .../unit/analysis-resource-budget.test.ts | 46 ++++++++++++++ .../unit/analyze-launch-branch-settle.test.ts | 47 ++++++++++++++ .../test/unit/analyze-worker-core.test.ts | 50 +++++++++++++++ 9 files changed, 246 insertions(+), 64 deletions(-) diff --git a/gitnexus/src/core/analysis-progress.ts b/gitnexus/src/core/analysis-progress.ts index fdd75bf78..5830024e3 100644 --- a/gitnexus/src/core/analysis-progress.ts +++ b/gitnexus/src/core/analysis-progress.ts @@ -27,6 +27,68 @@ export interface AnalysisPhaseProgress { modelDownloadPercent?: number; } +const publicEmbeddingProgress = ( + progress: EmbeddingJobProgress | undefined, +): EmbeddingJobProgress | undefined => { + if ( + !progress || + !['preparing', 'embedding', 'indexing', 'ready', 'preserved', 'failed', 'unverified'].includes( + progress.phase, + ) + ) + return undefined; + const safe: EmbeddingJobProgress = { phase: progress.phase }; + // Explicit numeric allowlist: the source may also contain internal paths, + // identities, credentials, or values that cannot cross JSON IPC. + for (const key of [ + 'totalUnits', + 'completedUnits', + 'reusedUnits', + 'pendingUnits', + 'failedUnits', + 'totalNodes', + 'completedNodes', + 'reusedNodes', + 'pendingNodes', + 'failedNodes', + 'chunks', + ] as const) { + const value = progress[key]; + if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) safe[key] = value; + } + if ( + progress.activeUnit === null || + (typeof progress.activeUnit === 'number' && + Number.isSafeInteger(progress.activeUnit) && + progress.activeUnit >= 0) + ) + safe.activeUnit = progress.activeUnit; + return safe; +}; + +/** The same allowlist protects worker IPC and public progress feeds. */ +export function projectAnalysisPhaseProgress( + progress: AnalysisPhaseProgress | undefined, +): AnalysisPhaseProgress { + const result: AnalysisPhaseProgress = {}; + if (!progress) return result; + if ( + progress.graphPhase && + ['pending', 'building', 'ready', 'failed'].includes(progress.graphPhase) + ) + result.graphPhase = progress.graphPhase; + const embeddingProgress = publicEmbeddingProgress(progress.embeddingProgress); + if (embeddingProgress) result.embeddingProgress = embeddingProgress; + if ( + typeof progress.modelDownloadPercent === 'number' && + Number.isFinite(progress.modelDownloadPercent) && + progress.modelDownloadPercent >= 0 && + progress.modelDownloadPercent <= 100 + ) + result.modelDownloadPercent = progress.modelDownloadPercent; + return result; +} + export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJobProgress { const counts = [ state.totalUnits, diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 95c81dd61..63a2f87e0 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -21,6 +21,8 @@ export interface AnalysisResourceRequest { /** Resume/sync can admit embeddings without reserving parser workers. */ includeParsing?: boolean; graphPoolBytes?: number; + /** Resident parent process, charged once per child phase. */ + supervisorBytes?: number; /** Estimated peak model/sidecar allocation; zero for an HTTP provider. */ modelMemoryMb: number; } @@ -29,6 +31,7 @@ export interface AnalysisResourceBudget { readonly totalBytes: number; readonly automatic: boolean; readonly headroomBytes: number; + readonly supervisorBytes: number; readonly mainHeapMb: number; readonly parse: { readonly workers: number; @@ -86,6 +89,9 @@ export function resolveAnalysisResourceBudget( } const totalBytes = bytes(totalMb, 'resource budget'); const headroomMb = Math.max(128, Math.ceil(totalMb * 0.12)); + const supervisorMb = Math.ceil( + integer(request.supervisorBytes ?? 0, 'supervisor reservation', 0) / MIB, + ); const graphPoolBytes = request.graphPoolBytes === undefined ? bytes(Math.max(GRAPH_POOL_MB, Math.min(4096, Math.floor(totalMb * 0.15))), 'graph pool') @@ -100,10 +106,14 @@ export function resolveAnalysisResourceBudget( // A supported 10 MiB source node needs its full text/AST working set. Larger // explicit budgets must be able to admit it without truncating its content. const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32))); - const embeddingFixedMb = headroomMb + graphPoolMb + VECTOR_POOL_MB + modelMb + queueMb; + const embeddingFixedMb = + headroomMb + supervisorMb + graphPoolMb + VECTOR_POOL_MB + modelMb + queueMb; const includeParsing = request.includeParsing !== false; const parseFixedMb = - headroomMb + graphPoolMb + (includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0); + headroomMb + + supervisorMb + + graphPoolMb + + (includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0); const maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); const mainHeapMb = request.mainHeapMb === undefined @@ -130,7 +140,7 @@ export function resolveAnalysisResourceBudget( `Requested ${desiredWorkers} parser workers exceeds the ${cpuWorkers}-worker CPU allowance`, ); } - const workerBudgetMb = totalMb - headroomMb - graphPoolMb - mainHeapMb; + const workerBudgetMb = totalMb - headroomMb - supervisorMb - graphPoolMb - mainHeapMb; const explicitWorkerHeapMb = request.workerHeapMb === undefined ? undefined @@ -157,7 +167,11 @@ export function resolveAnalysisResourceBudget( : (explicitWorkerHeapMb ?? Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB)); const parseReservedMb = - headroomMb + graphPoolMb + mainHeapMb + workers * (workerHeapMb + WORKER_NATIVE_MB); + headroomMb + + supervisorMb + + graphPoolMb + + mainHeapMb + + workers * (workerHeapMb + WORKER_NATIVE_MB); if ( (includeParsing && (workers < 1 || workerHeapMb < MIN_WORKER_HEAP_MB)) || parseReservedMb > totalMb @@ -171,6 +185,7 @@ export function resolveAnalysisResourceBudget( totalBytes, automatic: request.totalMb === undefined, headroomBytes: bytes(headroomMb, 'headroom'), + supervisorBytes: bytes(supervisorMb, 'supervisor reservation'), mainHeapMb, parse: Object.freeze({ workers, diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts index 840d18285..123c64d72 100644 --- a/gitnexus/src/server/analyze-launch.ts +++ b/gitnexus/src/server/analyze-launch.ts @@ -30,6 +30,7 @@ import type { RouteHookConfig } from '../core/ingestion/language-provider.js'; import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; import { isTerminalJobStatus, type JobManager } from './analyze-job.js'; import type { WorkerMessage } from './analyze-worker.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; const _require = createRequire(import.meta.url); @@ -343,7 +344,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { if (msg.type === 'progress') { jobManager.updateJob(job.id, { status: 'analyzing', - progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, + progress: { + ...projectAnalysisPhaseProgress(current.progress), + ...projectAnalysisPhaseProgress(msg), + phase: msg.phase, + percent: msg.percent, + message: msg.message, + }, }); } else if (msg.type === 'complete') { if (jobManager.applyPendingCancel(job.id)) { diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index b0e38405b..477d8910f 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -13,6 +13,7 @@ * worker — only the lightweight `analyze-worker-ipc` projection helper. */ import type { AnalyzeOptions } from '../core/run-analyze.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; // The IPC message protocol lives in a declarations-only leaf, NOT in the // `analyze-worker.ts` entry module: importing it from the entry made this // module depend on the very module that depends on it (#cycle). Erased at @@ -65,8 +66,14 @@ export async function runWorkerAnalysis( // writes); flushWAL still persists the index, process.exit reclaims handles. { ...options, skipNativeCloseOnExit: true }, { - onProgress: (phase, percent, message) => - deps.send({ type: 'progress', phase, percent, message }), + onProgress: (phase, percent, message, details) => + deps.send({ + type: 'progress', + phase, + percent, + message, + ...projectAnalysisPhaseProgress(details), + }), onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), }, ...bootstrapArgs, diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 9beb534e9..28bd9802c 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -24,6 +24,7 @@ * sites (`analyze-launch.ts`, tests) keep working unchanged. */ import type { AnalyzeOptions } from '../core/run-analyze.js'; +import type { AnalysisPhaseProgress } from '../core/analysis-progress.js'; import type { AnalyzeResultIpc } from './analyze-worker-ipc.js'; import type { PreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; @@ -41,7 +42,7 @@ export interface CancelMessage { export type ParentMessage = StartMessage | CancelMessage; -export interface ProgressMessage { +export interface ProgressMessage extends AnalysisPhaseProgress { type: 'progress'; phase: string; percent: number; diff --git a/gitnexus/src/server/ops-snapshot.ts b/gitnexus/src/server/ops-snapshot.ts index f29cadaf9..4ad9c0ede 100644 --- a/gitnexus/src/server/ops-snapshot.ts +++ b/gitnexus/src/server/ops-snapshot.ts @@ -13,46 +13,7 @@ import { } from './analyze-job.js'; import { publicRepoId } from './public-repo-id.js'; import { escapeRegExp } from './validation.js'; -import type { EmbeddingJobProgress } from '../core/analysis-progress.js'; - -const publicEmbeddingProgress = ( - progress: EmbeddingJobProgress | undefined, -): EmbeddingJobProgress | undefined => { - if ( - !progress || - !['preparing', 'embedding', 'indexing', 'ready', 'preserved', 'failed', 'unverified'].includes( - progress.phase, - ) - ) - return undefined; - const safe: EmbeddingJobProgress = { phase: progress.phase }; - // Explicit numeric allowlist: never spread the worker/job object onto a - // public feed. It may also contain source paths, identities, or credentials. - for (const key of [ - 'totalUnits', - 'completedUnits', - 'reusedUnits', - 'pendingUnits', - 'failedUnits', - 'totalNodes', - 'completedNodes', - 'reusedNodes', - 'pendingNodes', - 'failedNodes', - 'chunks', - ] as const) { - const value = progress[key]; - if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) safe[key] = value; - } - if ( - progress.activeUnit === null || - (typeof progress.activeUnit === 'number' && - Number.isSafeInteger(progress.activeUnit) && - progress.activeUnit >= 0) - ) - safe.activeUnit = progress.activeUnit; - return safe; -}; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; export interface OpsJobView { id: string; @@ -268,26 +229,12 @@ export const publicOpsProgress = ( progress: AnalyzeJobProgress, known?: Array, ): AnalyzeJobProgress => { - const result: AnalyzeJobProgress = { + return { phase: progress.phase, percent: progress.percent, message: redactPublicText(progress.message, known), + ...projectAnalysisPhaseProgress(progress), }; - if ( - progress.graphPhase && - ['pending', 'building', 'ready', 'failed'].includes(progress.graphPhase) - ) - result.graphPhase = progress.graphPhase; - const embeddingProgress = publicEmbeddingProgress(progress.embeddingProgress); - if (embeddingProgress) result.embeddingProgress = embeddingProgress; - if ( - typeof progress.modelDownloadPercent === 'number' && - Number.isFinite(progress.modelDownloadPercent) && - progress.modelDownloadPercent >= 0 && - progress.modelDownloadPercent <= 100 - ) - result.modelDownloadPercent = progress.modelDownloadPercent; - return result; }; export const serializeOpsJob = ( diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 1d92955f6..b9dfbedd4 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -8,6 +8,52 @@ const MIB = 1024 * 1024; const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; describe('analysis resource admission', () => { + it('charges a retained supervisor once per phase, independently of worker thread count', () => { + const fixed = { + ...request, + totalMb: 24576, + mainHeapMb: 8192, + workers: 2, + workerHeapMb: 1024, + graphPoolBytes: 1024 * MIB, + }; + const before = resolveAnalysisResourceBudget(fixed); + const after = resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 192 * MIB + 1 }); + expect(after.supervisorBytes).toBe(193 * MIB); + expect(after.mainHeapMb).toBe(before.mainHeapMb); + expect(after.parse.workers).toBe(before.parse.workers); + expect(after.parse.workerHeapMb).toBe(before.parse.workerHeapMb); + expect(after.parse.reservedBytes - before.parse.reservedBytes).toBe(193 * MIB); + expect(after.embedding.reservedBytes - before.embedding.reservedBytes).toBe(193 * MIB); + }); + + it('rejects an embedding-only process family whose supervisor exceeds its remaining capacity', () => { + const fixed = { + ...request, + totalMb: 1024, + mainHeapMb: 384, + graphPoolBytes: 256 * MIB, + modelMemoryMb: 0, + includeParsing: false, + }; + expect( + resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 160 * MIB }).embedding + .reservedBytes, + ).toBe(1024 * MIB); + expect(() => resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 161 * MIB })).toThrow( + 'cannot admit', + ); + }); + + it.each([NaN, Infinity, -1, 0.5])( + 'rejects an invalid supervisor reservation %s', + (supervisorBytes) => { + expect(() => resolveAnalysisResourceBudget({ ...request, supervisorBytes })).toThrow( + 'supervisor', + ); + }, + ); + it('bounds every phase separately and reuses parser capacity for embedding', () => { for (const memoryMb of [2048, 4096, 8192, 16384, 32768]) { for (const availableCpus of [1, 2, 4, 24]) { diff --git a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts index 02978ff4f..0bd1cff0e 100644 --- a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts +++ b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts @@ -144,6 +144,53 @@ describe('finalization gate follows the placement the run chose', () => { forkMock.mockReset(); }); + it('keeps graph readiness and embedding counters nonterminal across worker log messages', async () => { + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher()(job, REPO_PATH, {}); + const embeddingProgress = { + phase: 'embedding', + totalNodes: 3, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 1, + failedNodes: 0, + activeUnit: 0, + chunks: 2, + }; + child.emit('message', { + type: 'progress', + phase: 'embedding', + percent: 70, + message: 'Embedding', + graphPhase: 'ready', + embeddingProgress: { ...embeddingProgress, source: '/private/source', token: 1n }, + modelDownloadPercent: 100, + }); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(jobManager.getJob(job.id)?.progress).toMatchObject({ + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 100, + }); + expect(JSON.stringify(jobManager.getJob(job.id)?.progress)).not.toContain('/private'); + child.emit('message', { + type: 'progress', + phase: 'log', + percent: -1, + message: 'Native checkpoint', + }); + expect(jobManager.getJob(job.id)?.progress).toEqual({ + phase: 'log', + percent: -1, + message: 'Native checkpoint', + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 100, + }); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(backendInit).not.toHaveBeenCalled(); + }); + it('completes a branch sub-slot run, whose files are NOT in the flat slot', async () => { // Only `branches//` looks written. The flat slot is stale, so a gate // probing it would spin to the 60s timeout instead of settling here. diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index e69ec6586..fb64ebb2b 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -20,6 +20,7 @@ import { import type { AnalyzeResult } from '../../src/core/run-analyze.js'; import type { WorkerMessage } from '../../src/server/analyze-worker.js'; import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; +import type { AnalysisPhaseProgress } from '../../src/core/analysis-progress.js'; import { IndexLockTimeoutError, type LockRecord } from '../../src/storage/index-lock.js'; const baseResult: AnalyzeResult = { @@ -36,6 +37,55 @@ const okRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => baseResul const okFinalize: WorkerAnalysisDeps['assertAnalysisFinalized'] = vi.fn(async () => undefined); const alwaysClaim: WorkerAnalysisDeps['claimTerminal'] = () => true; +it('carries bounded phase progress over worker IPC without serializing internal fields', async () => { + const send = vi.fn<(msg: WorkerMessage) => void>((message) => { + JSON.stringify(message); + }); + const embeddingProgress = { + phase: 'embedding' as const, + totalNodes: 3, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 1, + failedNodes: 0, + activeUnit: 0, + chunks: 2, + }; + const details = { + graphPhase: 'ready', + embeddingProgress: { ...embeddingProgress, identity: 1n }, + modelDownloadPercent: 25, + sourcePath: '/private/source', + native: 1n, + } as unknown as AnalysisPhaseProgress; + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async (_repo, _options, callbacks) => { + callbacks.onProgress('embedding', 70, 'Embedding current selection', details); + return baseResult; + }), + assertAnalysisFinalized: okFinalize, + claimTerminal: alwaysClaim, + send, + }, + ); + expect( + send.mock.calls.filter(([message]) => message.type === 'progress').map(([message]) => message), + ).toEqual([ + { + type: 'progress', + phase: 'embedding', + percent: 70, + message: 'Embedding current selection', + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 25, + }, + ]); +}); + describe('prepared graph worker handoff', () => { const handoff = { version: 1 as const, From 7b7c7d52da352de243ebeb4ab7d326ed6bc1fe42 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 12:12:27 +0000 Subject: [PATCH 23/67] fix(storage): retain owned analysis generations Keep private generation files in place through shared-store preparation and publication. Retain prepared publications and pending embedding work across the unlocked process handoff, including work in branch slots. Refs #3598 --- gitnexus/src/core/shared-store-analyze.ts | 11 +- .../src/storage/shared-store-lifecycle.ts | 57 +++++- gitnexus/src/storage/shared-store.ts | 9 + .../unit/shared-store-generations.test.ts | 164 ++++++++++++++++++ 4 files changed, 232 insertions(+), 9 deletions(-) create mode 100644 gitnexus/test/unit/shared-store-generations.test.ts diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 5e06a68fa..58c95ee0e 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -41,6 +41,7 @@ import { cloneStoreKey, commitGraphDir, resolveGraphPath, + resolveGraphWritePath, resolveSharedStore, sharedStoreLayout, storeRootOfCheckoutSlot, @@ -316,7 +317,7 @@ export const ensurePrivateSharedGraph = async ( log: Log, opts: { copy?: boolean } = {}, ): Promise => { - const own = path.join(slot, LBUG_DIRECTORY); + const own = resolveGraphWritePath(slot); const pointed = resolveGraphPath(slot); if (pointed === own) return true; const meta = await loadMeta(slot); @@ -384,7 +385,7 @@ export const publishSharedGraph = async ( const slot = layout.checkoutSlot; const meta = await loadMeta(slot); if (!meta) return; - const own = path.join(slot, LBUG_DIRECTORY); + const own = resolveGraphWritePath(slot); // A graph built while files were dirty must not become shared until analyze // has reconciled those paths with disk and cleared them from the coverage receipt. @@ -481,7 +482,11 @@ export const publishSharedGraph = async ( meta.graphPath = targetGraph; await saveMeta(slot, meta); } - } else if (meta.graphPath !== undefined && (await exists(own))) { + } else if ( + meta.graphPath !== undefined && + path.isAbsolute(meta.graphPath) && + (await exists(own)) + ) { delete meta.graphPath; await saveMeta(slot, meta); } diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index 081441cc7..413e7c8b0 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -105,6 +105,48 @@ const loadMetaStrict = async (slot: string): Promise => { return null; }; +/** Each branch owns its own publication and vector work under the member's + * ordinary slot lock. Never follow a symlink while deciding what to collect. */ +async function memberMetadataDirectories(slot: string): Promise { + const directories = [slot]; + const branches = path.join(slot, 'branches'); + try { + const stat = await fs.lstat(branches); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Cannot collect a shared member with an unverified branches directory'); + for (const entry of await fs.readdir(branches, { withFileTypes: true })) { + if (entry.isSymbolicLink()) + throw new Error('Cannot collect a shared member with a symlinked branch'); + if (entry.isDirectory() && !entry.name.startsWith('.')) + directories.push(path.join(branches, entry.name)); + } + } catch (error) { + if (!isMissingFilesystemError(error)) throw error; + } + return directories; +} + +/** The writer releases its lock before an exit-only supervisor reacquires it. + * Its durable intent still owns the files throughout that gap and a restart. + * A malformed receipt is retained evidence, never permission to delete it. */ +async function hasRetainedAnalysis(slot: string): Promise { + for (const directory of await memberMetadataDirectories(slot)) { + try { + await fs.lstat(path.join(directory, 'analysis-publication.json')); + return true; + } catch (error) { + if (!isMissingFilesystemError(error)) throw error; + } + const metadata = await loadMetaStrict(directory); + if ( + metadata && + (Object.hasOwn(metadata, 'embeddingJob') || Object.hasOwn(metadata, 'embeddingCheckpoint')) + ) + return true; + } + return false; +} + /** * Member slots that no registry entry uses any more: the checkout directory is * gone, or its entry moved elsewhere (`--no-share`, sharing turned off). The @@ -121,7 +163,7 @@ const orphanMembers = async (slots: string[]): Promise> => { const meta = await loadMeta(slot); if (!meta?.repoPath) continue; if (!existsSync(meta.repoPath)) { - orphans.add(slot); + if (!(await hasRetainedAnalysis(slot))) orphans.add(slot); continue; } if (!entries) continue; @@ -130,7 +172,7 @@ const orphanMembers = async (slots: string[]): Promise> => { !entry || !registryPathEquals(canonicalizePath(entry.storagePath), canonicalizePath(slot)) ) { - orphans.add(slot); + if (!(await hasRetainedAnalysis(slot))) orphans.add(slot); } } return orphans; @@ -168,8 +210,9 @@ export const reclaimSharedStoreLocked = async ( // whose lock is free, so the preview matches what --force would do. let lock: IndexLockHandle; try { - // A preview deletes nothing, not even the staging files the lock sweeps. - lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: !opts.dryRun }); + // Recheck durable work under ownership before deleting anything. In + // particular, acquiring the lock must not sweep a just-prepared graph. + lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: false }); } catch { orphans.delete(slot); continue; @@ -199,8 +242,10 @@ export const reclaimSharedStoreLocked = async ( slots = slots.filter((slot) => !orphans.has(slot)); } for (const slot of slots) { - const graphPath = (await loadMetaStrict(slot))?.graphPath; - if (graphPath) referenced.add(path.dirname(path.resolve(graphPath))); + for (const directory of await memberMetadataDirectories(slot)) { + const graphPath = (await loadMetaStrict(directory))?.graphPath; + if (graphPath) referenced.add(path.dirname(path.resolve(directory, graphPath))); + } } for (const name of await listDirStrict(commitsDir)) { diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index 3654e8615..f3c72c1b4 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -298,6 +298,15 @@ export const resolveGraphPath = (storagePath: string): string => { return valid ? graph : own; }; +/** A private generation remains its own write target. Only an immutable + * shared commit graph needs a new slot-local file before mutation. */ +export const resolveGraphWritePath = (storagePath: string): string => { + const graph = resolveGraphPath(storagePath); + return path.dirname(path.resolve(graph)) === path.resolve(storagePath) + ? graph + : path.join(storagePath, LBUG_DIRECTORY); +}; + /** * The store slot named by `/.gitnexus/store.json`, or null. The * recorded slot must be this checkout's own slot under the stores directory, diff --git a/gitnexus/test/unit/shared-store-generations.test.ts b/gitnexus/test/unit/shared-store-generations.test.ts new file mode 100644 index 000000000..12c8461a5 --- /dev/null +++ b/gitnexus/test/unit/shared-store-generations.test.ts @@ -0,0 +1,164 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + ensurePrivateSharedGraph, + publishSharedGraph, +} from '../../src/core/shared-store-analyze.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { reclaimSharedStore } from '../../src/storage/shared-store-lifecycle.js'; +import { + resolveGraphPath, + resolveGraphWritePath, + sharedStoreLayout, +} from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +describe('shared-store logical generations', () => { + let root: string; + let repo: string; + let head: string; + let slot: string; + let layout: ReturnType; + const generation = `lbug.generation.${'a'.repeat(32)}`; + const git = (...args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8', stdio: 'pipe' }).trim(); + const metadata = (extra: Partial = {}): RepoMeta => ({ + repoPath: repo, + storagePath: slot, + lastCommit: head, + indexedAt: '2026-11-04T00:00:00Z', + parseCacheIdentity: { base: 'fixture-parser' }, + ...extra, + }); + + beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-shared-generations-'))); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + repo = path.join(root, 'repo'); + await fs.mkdir(repo); + git('init', '-q', '-b', 'main'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '--allow-empty', '-qm', 'fixture'); + head = git('rev-parse', 'HEAD'); + layout = sharedStoreLayout('fixture-0123456789ab', repo); + slot = layout.checkoutSlot; + await fs.mkdir(slot, { recursive: true }); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it.each([true, false])( + 'keeps an owned generation in place without copying or dropping its pointer (copy=%s)', + async (copy) => { + const file = path.join(slot, generation); + await fs.writeFile(file, 'private graph'); + await saveMeta(slot, metadata({ graphPath: generation })); + const before = await fs.stat(file); + const copyFile = vi.spyOn(fs, 'copyFile'); + const log = vi.fn(); + expect(await ensurePrivateSharedGraph(slot, log, { copy })).toBe(true); + expect(resolveGraphPath(slot)).toBe(file); + expect((await fs.stat(file)).ino).toBe(before.ino); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(copyFile).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + }, + ); + + it('routes only immutable shared graphs to a new private write target', async () => { + const commitGraph = path.join(layout.commitsDir, `${head}-deadbeef`, 'lbug'); + await saveMeta(slot, metadata({ graphPath: commitGraph })); + expect(resolveGraphWritePath(slot)).toBe(path.join(slot, 'lbug')); + await fs.writeFile(path.join(slot, generation), 'private graph'); + await saveMeta(slot, metadata({ graphPath: generation })); + expect(resolveGraphWritePath(slot)).toBe(path.join(slot, generation)); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await fs.writeFile(path.join(branch, generation), 'branch graph'); + await saveMeta(branch, metadata({ graphPath: generation })); + expect(resolveGraphWritePath(branch)).toBe(path.join(branch, generation)); + }); + + it('publishes a completed private generation by moving its file, preserving the inode', async () => { + const file = path.join(slot, generation); + await fs.writeFile(file, 'completed graph'); + const before = await fs.stat(file); + await saveMeta(slot, metadata({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + const published = resolveGraphPath(slot); + expect(path.dirname(path.dirname(published))).toBe(layout.commitsDir); + expect((await fs.stat(published)).ino).toBe(before.ino); + expect(await fs.readFile(published, 'utf8')).toBe('completed graph'); + }); + + it('keeps an unshareable private generation pointer intact', async () => { + await fs.writeFile(path.join(slot, generation), 'dirty graph'); + await fs.writeFile(path.join(slot, 'lbug'), 'obsolete canonical graph'); + await fs.writeFile(path.join(repo, 'untracked.ts'), 'export const changed = true;'); + await saveMeta(slot, metadata({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + expect((await loadMeta(slot))?.graphPath).toBe(generation); + expect(resolveGraphPath(slot)).toBe(path.join(slot, generation)); + }); + + it.each(['prepared', 'pending', 'branch-prepared', 'branch-pending'])( + 'retains an orphan slot with %s analysis work', + async (kind) => { + const missingRepo = path.join(root, 'removed-checkout'); + await saveMeta(slot, metadata({ repoPath: missingRepo })); + const work = kind.startsWith('branch-') ? path.join(slot, 'branches', 'feature') : slot; + await fs.mkdir(work, { recursive: true }); + if (kind.endsWith('prepared')) { + // Even an unreadable/incomplete publication receipt is retained evidence. + await fs.writeFile(path.join(work, 'analysis-publication.json'), '{}'); + } else { + await saveMeta( + work, + metadata({ + repoPath: missingRepo, + embeddingJob: { + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + }, + }), + ); + } + await fs.writeFile(path.join(work, 'paid-work'), 'retained'); + const result = await reclaimSharedStore(layout.root, { gc: true }); + expect(result.droppedMembers).toEqual([]); + expect(await fs.readFile(path.join(work, 'paid-work'), 'utf8')).toBe('retained'); + await fs.rm(path.join(work, 'analysis-publication.json'), { force: true }); + await saveMeta(work, metadata({ repoPath: missingRepo })); + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([slot]); + await expect(fs.stat(slot)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('counts retained branch references before collecting shared graphs', async () => { + const graph = path.join(layout.commitsDir, `${head}-deadbeef`, 'lbug'); + await fs.mkdir(path.dirname(graph), { recursive: true }); + await fs.writeFile(graph, 'retained graph'); + await saveMeta(slot, metadata()); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await saveMeta(branch, metadata({ graphPath: graph })); + expect((await reclaimSharedStore(layout.root)).removed).toEqual([]); + expect(await fs.readFile(graph, 'utf8')).toBe('retained graph'); + }); + + it('aborts collection when branch ownership cannot be inspected', async () => { + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + await fs.writeFile(path.join(slot, 'branches'), 'unverified branch state'); + await expect(reclaimSharedStore(layout.root, { gc: true })).rejects.toThrow( + 'unverified branches directory', + ); + expect(await fs.readFile(path.join(slot, 'branches'), 'utf8')).toBe('unverified branch state'); + }); +}); From a0e026369117e41c0521dedafa8dcdbaed0f0152 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 12:32:40 +0000 Subject: [PATCH 24/67] fix(embeddings): share pinned vector readers Share immutable vector file identities so sibling hard links retain one native reservation. Keep the handle alive until its final reader closes, including failed initialization and replacement of another reader's path. Refs #3598 --- gitnexus/src/core/lbug/pool-adapter.ts | 71 +++++++++----- .../embedding-vector-store.test.ts | 42 +++++++++ gitnexus/test/unit/lbug-vector-pool.test.ts | 92 ++++++++++++++++++- 3 files changed, 180 insertions(+), 25 deletions(-) diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index 23d845bbe..ab64ae9b1 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -60,6 +60,9 @@ interface PoolEntry { }>; lastUsed: number; dbPath: string; + /** Immutable vector hard links share one native pool, independently of the + * path each checkout retains. Graph and injected handles remain path-keyed. */ + dbCacheKey?: string; /** Filesystem identity of the on-disk DB at open time. When `analyze` * rebuilds or mutates the index, this diverges from the current file and * initLbug re-opens the pool onto the new file instead of serving the @@ -139,9 +142,8 @@ export function addPoolCloseListener(listener: PoolCloseListener): () => void { } /** - * Shared Database cache keyed by resolved dbPath. - * Multiple repoIds pointing to the same path share one native Database - * object to avoid exhausting the buffer manager's mmap budget. + * Shared Database cache keyed by graph path or immutable vector file identity. + * Sibling hard links must not multiply a sealed artifact's native reservation. */ interface SharedDB { db: lbug.Database; @@ -445,7 +447,8 @@ async function closeOne(repoId: string): Promise { // Only close the Database when no other repoIds reference it. // External databases (injected via initLbugWithDb) are never closed here — // the core adapter owns them and handles their lifecycle. - const shared = dbCache.get(entry.dbPath); + const cacheKey = entry.dbCacheKey ?? entry.dbPath; + const shared = dbCache.get(cacheKey); if (shared) { shared.refCount--; if (shared.refCount === 0) { @@ -463,7 +466,7 @@ async function closeOne(repoId: string): Promise { // subsequent reopen (evictLRU / the "idle & changed" path below) must not // race. See the awaited call site in evictLRU for the full rationale. await shared.db.close().catch(() => {}); - dbCache.delete(entry.dbPath); + dbCache.delete(cacheKey); } } } @@ -855,12 +858,18 @@ const initLbugInner = async ( // freshness THROUGH initLbug (rather than calling closeLbug directly) get // this guard for free; that is why LocalBackend delegates here (#2614). if (existing.checkedOut > 0) return { status: 'done', reopened: false }; - // Closing only this consumer would leave the shared native handle alive + // For graph paths, closing only this consumer leaves the shared handle alive // and doInitLbug would attach eight new connections to that same old // generation. Keep this pool intact until the other consumers release it; // its opened identity stays divergent so the next init can retry then. - const shared = dbCache.get(existing.dbPath); - if (existing.dbPath === dbPath && shared && shared.db === existing.db && shared.refCount > 1) { + const shared = dbCache.get(existing.dbCacheKey ?? existing.dbPath); + if ( + !artifactOptions && + existing.dbPath === dbPath && + shared && + shared.db === existing.db && + shared.refCount > 1 + ) { return { status: 'done', reopened: false }; } // Awaited: see the rationale on the evictLRU call site in doInitLbug below. @@ -876,24 +885,38 @@ function sameArtifactOptions(a?: VectorPoolOptions, b?: VectorPoolOptions): bool ); } -/** Published vectors are immutable: readers never quarantine, replay, or repair - * their writer's sidecars. An unfinished artifact must stay unavailable. */ -async function openVectorReadOnlyDatabase( - dbPath: string, - options: VectorPoolOptions, -): Promise { - const stat = await fs.lstat(dbPath); +/** Validate every retained path before sharing a native handle. No content + * hash or copy: sealed files can be pinned by hard links without extra pools. + * An unavailable inode identity falls back to the existing path isolation. */ +async function vectorDatabaseKey(dbPath: string): Promise { + const stat = await fs.lstat(dbPath, { bigint: true }); if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('Vector artifact must be a regular file'); for (const suffix of ['.wal', '.shadow']) { try { const sidecar = await fs.lstat(dbPath + suffix); - if (sidecar.size || sidecar.isSymbolicLink()) + if (sidecar.size || !sidecar.isFile() || sidecar.isSymbolicLink()) throw new Error('Published vector artifact has unfinished native writes'); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } } + // NUL cannot occur in a filesystem path, so these keys cannot collide with + // the graph cache. ctime changes when another hard link is added; mtime does not. + return typeof stat.ino === 'bigint' && + stat.ino > 0n && + typeof stat.dev === 'bigint' && + stat.dev > 0n + ? `vector\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}` + : `vector\0path\0${dbPath}`; +} + +/** Called after vectorDatabaseKey validates the immutable file and sidecars. + * Readers never quarantine, replay, or repair their writer's sidecars. */ +async function openVectorReadOnlyDatabase( + dbPath: string, + options: VectorPoolOptions, +): Promise { silenceStdout(); let db: lbug.Database | undefined; try { @@ -938,9 +961,10 @@ async function doInitLbug( // let a 6th native open race a still-resident victim's checkpoint. await evictLRU(artifactOptions?.kind); - // Reuse an existing native Database if another repoId already opened this path. - // This prevents buffer manager exhaustion from multiple mmap regions on the same file. - let shared = dbCache.get(dbPath); + // Immutable vector aliases share the reservation even when their checkout + // paths differ. Graph databases retain their existing path-based lifecycle. + const cacheKey = artifactOptions ? await vectorDatabaseKey(dbPath) : dbPath; + let shared = dbCache.get(cacheKey); if (shared && !sameArtifactOptions(shared.artifactOptions, artifactOptions)) throw new Error('A shared database has a different resource reservation'); if (shared && !shared.external && shared.dbIdentity) { @@ -976,7 +1000,7 @@ async function doInitLbug( dbIdentity: await statDbIdentity(dbPath), artifactOptions, }; - dbCache.set(dbPath, shared); + dbCache.set(cacheKey, shared); } catch (err: unknown) { const lastError = err instanceof Error ? err : new Error(String(err)); if (artifactOptions) throw lastError; @@ -997,7 +1021,7 @@ async function doInitLbug( ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath), }; - dbCache.set(dbPath, shared); + dbCache.set(cacheKey, shared); } catch (retryErr) { throw new Error( `LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` + @@ -1044,7 +1068,7 @@ async function doInitLbug( await Promise.allSettled(available.map((connection) => connection.close())); shared.refCount--; if (shared.refCount === 0) { - dbCache.delete(dbPath); + dbCache.delete(cacheKey); if (!shared.external) await db.close().catch(() => {}); } throw error; @@ -1074,6 +1098,7 @@ async function doInitLbug( waiters: [], lastUsed: Date.now(), dbPath, + dbCacheKey: cacheKey, dbIdentity, closed: false, artifactOptions, @@ -1169,7 +1194,7 @@ export async function ensureVectorExtension(repoId: string): Promise { throw new Error(`LadybugDB not initialized for repo "${repoId}". Call initLbug first.`); } - const shared = dbCache.get(entry.dbPath); + const shared = dbCache.get(entry.dbCacheKey ?? entry.dbPath); if (!shared) { throw new Error(`LadybugDB shared handle is unavailable for repo "${repoId}".`); } diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 94c63193a..f80c6810b 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -825,6 +825,48 @@ describe('separate durable vector artifacts', () => { }); describe('published vector readers', () => { + it('shares one native database across independently pinned vector files', async () => { + const options = await setup(); + const id = 'Function:source.ts:alpha'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.seal(1, 1); + await store.close(); + const first = path.join(options.directory, 'vectors'); + const second = path.join(options.directory, 'sibling-vectors'); + await fs.link(first, second); + const close = vi.spyOn(lbug.Database.prototype, 'close'); + const reservation = { kind: 'vector' as const, bufferPoolBytes: 64 * 1024 * 1024, threads: 1 }; + try { + await readerPool.initLbug('vector-first', first, reservation); + await readerPool.initLbug('vector-second', second, reservation); + await readerPool.closeLbug('vector-first'); + expect(close).not.toHaveBeenCalled(); + // POSIX can unlink the retired path while its immutable inode is open. + // Windows may retain that path until the last reader releases it. + if (process.platform !== 'win32') await fs.unlink(first); + expect( + await readerPool.executeQuery( + 'vector-second', + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS id', + ), + ).toEqual([{ id }]); + await readerPool.closeLbug('vector-second'); + expect(close).toHaveBeenCalledOnce(); + await readerPool.initLbug('vector-second', second, reservation); + expect( + await readerPool.executeQuery( + 'vector-second', + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS id', + ), + ).toEqual([{ id }]); + } finally { + await readerPool.closeLbug(); + close.mockRestore(); + } + }); + it('publishes a native FTS graph before opening the independent vector writer', async (context) => { temporary = await createTempDir(); vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); diff --git a/gitnexus/test/unit/lbug-vector-pool.test.ts b/gitnexus/test/unit/lbug-vector-pool.test.ts index 259140fa6..1f1bc6a26 100644 --- a/gitnexus/test/unit/lbug-vector-pool.test.ts +++ b/gitnexus/test/unit/lbug-vector-pool.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ create: vi.fn(), fts: vi.fn(), + vector: vi.fn(), threads: vi.fn(), connections: [] as Array<{ close: ReturnType }>, })); @@ -26,13 +27,18 @@ vi.mock('@ladybugdb/core', () => ({ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ isReadOnlyDbError: () => false, loadFTSExtension: mocks.fts, - loadVectorExtension: async () => false, + loadVectorExtension: mocks.vector, })); vi.mock('../../src/core/lbug/lbug-config.js', async (original) => ({ ...(await original()), createLbugDatabase: mocks.create, })); -import { closeLbug, initLbug, isLbugReady } from '../../src/core/lbug/pool-adapter.js'; +import { + closeLbug, + initLbug, + isLbugReady, + ensureVectorExtension, +} from '../../src/core/lbug/pool-adapter.js'; let directory: string; const options = { kind: 'vector' as const, bufferPoolBytes: 64 * 1024 * 1024, threads: 2 }; @@ -49,6 +55,7 @@ beforeEach(async () => { close: vi.fn(async () => {}), })); mocks.fts.mockResolvedValue(true); + mocks.vector.mockResolvedValue(false); mocks.threads.mockResolvedValue(undefined); directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-pool-')); }); @@ -58,6 +65,87 @@ afterEach(async () => { }); describe('vector pool isolation', () => { + it('shares one native reservation across hard links until the last reader closes', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledOnce(); + await ensureVectorExtension('first'); + await ensureVectorExtension('second'); + expect(mocks.vector).toHaveBeenCalledOnce(); + const database = mocks.create.mock.results[0].value; + await closeLbug('first'); + expect(database.close).not.toHaveBeenCalled(); + expect(isLbugReady('second')).toBe(true); + await closeLbug('second'); + expect(database.close).toHaveBeenCalledOnce(); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + + it('keeps independent files separate even when their bytes match', async () => { + await initLbug('first', await file('first'), options); + await initLbug('second', await file('second'), options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + + it('reopens a replaced vector path without closing a sibling on the previous inode', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await initLbug('second', second, options); + const original = mocks.create.mock.results[0].value; + await fs.rename(await file('replacement'), second); + expect(await initLbug('second', second, options)).toBe(true); + expect(mocks.create).toHaveBeenCalledTimes(2); + expect(original.close).not.toHaveBeenCalled(); + await closeLbug('first'); + expect(original.close).toHaveBeenCalledOnce(); + expect(isLbugReady('second')).toBe(true); + }); + + it('rejects a different reservation through another link to the same vectors', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await expect( + initLbug('second', second, { ...options, bufferPoolBytes: options.bufferPoolBytes * 2 }), + ).rejects.toThrow('different resource reservation'); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + + it('retains a shared native handle when another reader fails to initialize', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + const database = mocks.create.mock.results[0].value; + mocks.threads.mockRejectedValueOnce(new Error('thread configuration failed')); + await expect(initLbug('second', second, options)).rejects.toThrow( + 'thread configuration failed', + ); + expect(database.close).not.toHaveBeenCalled(); + expect(isLbugReady('first')).toBe(true); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + + it('validates a new link before reusing an already open native handle', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await fs.writeFile(second + '.wal', 'unfinished'); + await expect(initLbug('second', second, options)).rejects.toThrow('unfinished native writes'); + expect(mocks.create).toHaveBeenCalledOnce(); + expect(isLbugReady('first')).toBe(true); + expect(await fs.readFile(second + '.wal', 'utf8')).toBe('unfinished'); + }); + it('retains the graph-cache capacity while bounding vector residents independently', async () => { for (let i = 0; i < 5; i++) await initLbug(`graph-${i}`, await file(`graph-${i}`)); for (let i = 0; i < 6; i++) await initLbug(`vector-${i}`, await file(`vector-${i}`), options); From ecfda2cd3c8642a52f549bdc0bd9d16f7926f64d Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 13:14:01 +0000 Subject: [PATCH 25/67] feat(storage): share completed vector artifacts Pin immutable vectors independently in checkout slots and shared graph snapshots. Keep mutable jobs with their owning checkout; resume interrupted adoption without overwriting unverified files. Refs #3598 --- gitnexus/src/core/shared-store-analyze.ts | 151 ++++++++++++- gitnexus/src/storage/embedding-job.ts | 1 + .../src/storage/shared-vector-artifact.ts | 190 ++++++++++++++++ .../unit/shared-store-generations.test.ts | 207 ++++++++++++++++++ .../test/unit/shared-vector-artifact.test.ts | 159 ++++++++++++++ 5 files changed, 696 insertions(+), 12 deletions(-) create mode 100644 gitnexus/src/storage/shared-vector-artifact.ts create mode 100644 gitnexus/test/unit/shared-vector-artifact.test.ts diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 58c95ee0e..717873b21 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -37,6 +37,8 @@ import { type RegistryEntry, } from '../storage/repo-manager.js'; import { isMissingFilesystemError, loadMeta, type RepoMeta } from '../storage/repo-meta.js'; +import { retainSharedVectorArtifact } from '../storage/shared-vector-artifact.js'; +import { selectVectorGeneration } from '../storage/vector-generation.js'; import { cloneStoreKey, commitGraphDir, @@ -104,11 +106,117 @@ export const featureKeyOf = (meta: RepoMeta): string => { for (const [k, v] of Object.entries(meta)) { if (!FEATURE_KEY_EXCLUDED.has(k)) kept[k] = v; } - // Embedding presence changes graph content but lives only in `stats`. - kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0; + if (hasSeparateGraphGeneration(meta)) { + // Logical IDs and separate vector work do not change the graph's facts. + // Keep all unknown generation/capability fields in the denylist hash. + const generation: Record = { ...meta.graphGeneration }; + delete generation.id; + kept.graphGeneration = generation; + for (const key of ['embeddingJob', 'vectorGeneration', 'legacyVectorSource', 'embeddingDims']) + delete kept[key]; + if (meta.capabilities) { + const capabilities: Record = { ...meta.capabilities }; + delete capabilities.vectorSearch; + kept.capabilities = capabilities; + } + } else { + // Legacy embeddings still live inside the graph database. + kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0; + } return createHash('sha256').update(stableStringify(kept)).digest('hex').slice(0, 16); }; +function hasSeparateGraphGeneration(meta: RepoMeta): boolean { + const generation = meta.graphGeneration; + return ( + generation?.version === 1 && + /^[a-f0-9]{32}$/.test(generation.id) && + /^[a-f0-9]{64}$/.test(generation.coveredContent) + ); +} + +/** Share completed immutable files only. The original slot keeps its mutable + * job and receipts; siblings never inherit ownership of that paid work. */ +async function retainSharedMetadata( + source: string, + destination: string, + meta: RepoMeta, + allowCopy = false, +): Promise { + if ( + !meta.graphGeneration && + !meta.embeddingJob && + !meta.vectorGeneration && + !meta.legacyVectorSource + ) + return { ...meta }; + if (!hasSeparateGraphGeneration(meta)) + throw new Error('Cannot share an unverified logical graph generation'); + const exported = { ...meta }; + delete exported.embeddingJob; + delete exported.embeddingCheckpoint; + delete exported.legacyVectorSource; + const selected = selectVectorGeneration( + source, + exported, + resolveGraphPath(source), + Number.MAX_SAFE_INTEGER, + ); + const vector = meta.vectorGeneration; + const active = meta.embeddingJob?.artifactIdentity ?? meta.embeddingJob?.identity; + const completed = + vector && + active !== (vector.artifactIdentity ?? vector.identity) && + (selected.kind === 'ready' || + (selected.kind === 'unavailable' && selected.reason === 'embeddings-empty')); + if (completed) { + await retainSharedVectorArtifact(source, destination, vector, allowCopy); + } else { + delete exported.vectorGeneration; + delete exported.embeddingDims; + } + exported.stats = { ...meta.stats, embeddings: completed ? vector.chunks : 0 }; + if (meta.capabilities) { + const exactScanLimit = meta.capabilities.vectorSearch.exactScanLimit; + const readable = + completed && + vector.chunks > 0 && + (vector.indexKind === 'hnsw' || vector.chunks <= exactScanLimit); + exported.capabilities = { + ...meta.capabilities, + vectorSearch: { + provider: completed && vector.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: readable + ? vector.indexKind === 'hnsw' + ? 'vector-index' + : 'exact-scan' + : 'unavailable', + exactScanLimit, + coverage: completed + ? vector.graphGeneration === meta.graphGeneration?.id + ? 'current' + : 'preserved' + : 'unknown', + ...(!readable + ? { + reason: + completed && vector.chunks ? 'vector-index-pending' : 'embeddings-unavailable', + } + : {}), + ...(completed ? { sourceGraphGeneration: vector.graphGeneration } : {}), + }, + }; + } + return exported; +} + +/** Commit metadata intentionally omits checkout-only required RepoMeta fields. */ +function commitMetadata(meta: RepoMeta): RepoMeta { + const result: Record = { ...meta }; + for (const field of CHECKOUT_FIELDS) delete result[field]; + return result as unknown as RepoMeta; +} + const exists = (p: string): Promise => fs.access(p).then( () => true, @@ -205,7 +313,7 @@ const seedFromLocalIndex = async ( source: string, log: Log, ): Promise => { - const sourceGraph = path.join(source, LBUG_DIRECTORY); + let sourceGraph = resolveGraphPath(source); const usable = (m: RepoMeta | null): m is RepoMeta => !!m && !m.incrementalInProgress && @@ -220,13 +328,22 @@ const seedFromLocalIndex = async ( } let meta: RepoMeta; try { - if (lock.lockFree || (await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false; + if (lock.lockFree) return false; // Re-read under the lock: an analyze that finished while this waited // rewrote both, and the copied graph must match the saved metadata. const locked = await loadMeta(source); if (!usable(locked)) return false; - meta = locked; + sourceGraph = resolveGraphPath(source); + if ((await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false; + if ( + locked.embeddingJob && + registryPathEquals(canonicalizePath(locked.repoPath), canonicalizePath(repoPath)) + ) + throw new Error( + "Finish this checkout's pending embedding job in its original storage before shared-store adoption", + ); await fs.mkdir(slot, { recursive: true }); + meta = await retainSharedMetadata(source, slot, locked, true); try { await cloneGraphFile(sourceGraph, path.join(slot, LBUG_DIRECTORY)); } catch (err) { @@ -280,13 +397,18 @@ export const seedSharedSlot = async ( (await withStoreLock(layout, 'publish', async () => { const graph = path.join(seed.dir, LBUG_DIRECTORY); if (!(await exists(graph))) return false; + // Vector publication can advance this snapshot after pickSeed. Re-read + // while the publish lock also prevents its source from being collected. + const current = await loadMeta(seed.dir); + if (!current) return false; await fs.mkdir(layout.checkoutSlot, { recursive: true }); + const retained = await retainSharedMetadata(seed.dir, layout.checkoutSlot, current); const meta: RepoMeta = { - ...seed.meta, + ...retained, repoPath, storagePath: layout.checkoutSlot, graphPath: graph, - lastCommit: upToDate ? seed.meta.lastCommit : '', + lastCommit: upToDate && canReuseGraph(current) ? current.lastCommit : '', }; delete meta.incrementalInProgress; await saveMeta(layout.checkoutSlot, meta); @@ -398,6 +520,7 @@ export const publishSharedGraph = async ( // FTS-only repair preserves the graph's existing parser provenance. meta.parseCacheIdentity !== undefined && !meta.incrementalInProgress && + !meta.graphWriteCollapsed && builtClean && // Embeddings still owed stay with this checkout, which finishes them; a // commit graph never changes, so a shared copy would stay short for good. @@ -422,8 +545,9 @@ export const publishSharedGraph = async ( const ownIsBetter = published && (!targetMeta || - !!targetMeta.embeddingCheckpoint || - (targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)) && + (!hasSeparateGraphGeneration(meta) && + (!!targetMeta.embeddingCheckpoint || + (targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)))) && (await exists(own)); if (ownIsBetter) { published = false; @@ -432,22 +556,25 @@ export const publishSharedGraph = async ( ); } else if (published) { try { + if (hasSeparateGraphGeneration(meta) && meta.vectorGeneration && !meta.embeddingJob) { + const retained = await retainSharedMetadata(slot, target, meta); + if (retained.vectorGeneration) await saveMeta(target, commitMetadata(retained)); + } await wipeLbugDbFiles(own); } catch (err) { // An open reader (Windows) can block the delete. The analysis // already succeeded; keep the private graph and try next run. published = false; log( - `Shared store: could not drop the private graph (${(err as Error).message}); keeping it.`, + `Shared store: could not finalize the shared graph (${(err as Error).message}); keeping the existing graph.`, ); } } else if ((await exists(own)) && (await inspectLbugSidecars(own)).kind === 'clean') { await fs.mkdir(layout.commitsDir, { recursive: true }); const staging = path.join(layout.commitsDir, `.publish-${randomUUID()}`); await fs.mkdir(staging); - const commitMeta: Record = { ...meta }; - for (const field of CHECKOUT_FIELDS) delete commitMeta[field]; try { + const commitMeta = commitMetadata(await retainSharedMetadata(slot, staging, meta)); await fs.rename(own, path.join(staging, LBUG_DIRECTORY)); await fs.writeFile(path.join(staging, INDEX_METADATA_FILE), JSON.stringify(commitMeta)); await fs.rename(staging, target); diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index d66fc1aa3..42d292619 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -11,6 +11,7 @@ const RECORD_NAMES = new Set([ 'space.json', 'rebase.json', 'legacy.json', + 'retained.json', ]); const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; diff --git a/gitnexus/src/storage/shared-vector-artifact.ts b/gitnexus/src/storage/shared-vector-artifact.ts new file mode 100644 index 000000000..dfc4348c4 --- /dev/null +++ b/gitnexus/src/storage/shared-vector-artifact.ts @@ -0,0 +1,190 @@ +/** Retain a completed, immutable vector file in another owned slot. + * Caller holds the source/destination locks (or the shared publish lock). + * Pending artifacts, manifests, journals and mutable ownership never move. */ +import { createHash } from 'node:crypto'; +import { constants, type BigIntStats } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from './embedding-job.js'; +import { + assertOwnedVectorGeneration, + embeddingGenerationDirectory, + type PublishedVectorGeneration, +} from './vector-generation.js'; + +type FileIdentity = { dev: string; ino: string; size: string; mtimeNs: string }; +interface RetentionReceipt { + version: 1; + signature: string; + source: FileIdentity; + /** Absent until the entire retained file is durable. */ + file?: FileIdentity; +} +const facts = (stat: BigIntStats): FileIdentity => ({ + dev: String(stat.dev), + ino: String(stat.ino), + size: String(stat.size), + mtimeNs: String(stat.mtimeNs), +}); +const sameFile = (a: FileIdentity | undefined, b: FileIdentity): boolean => + !!a && (['dev', 'ino', 'size', 'mtimeNs'] as const).every((key) => a[key] === b[key]); + +async function completedFile(file: string): Promise { + const stat = await fs.lstat(file, { bigint: true }); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Retained vectors must be an owned regular file'); + for (const suffix of ['.wal', '.shadow']) { + try { + const sidecar = await fs.lstat(file + suffix); + if (!sidecar.isFile() || sidecar.isSymbolicLink() || sidecar.size) + throw new Error('Cannot retain vectors with unfinished native writes'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + return facts(stat); +} + +async function receipt(directory: string, signature: string): Promise { + const value = (await readEmbeddingJobRecord(directory, 'retained.json')) as RetentionReceipt; + if (!value || value.version !== 1 || value.signature !== signature || !value.source) + throw new Error('Unverified vector retention receipt; artifacts retained'); + return value; +} + +/** A crash can leave the directory before its first atomic receipt rename. + * Only unpublished receipt temporaries are ours to discard in that state. */ +async function recoverEmptyPreparation(directory: string): Promise { + await embeddingJobDirectory(directory); + const temporary = /^\.retained\.json\.[a-f0-9-]{36}\.tmp$/; + for await (const entry of await fs.opendir(directory)) { + const stat = await fs.lstat(path.join(directory, entry.name)); + if (!temporary.test(entry.name) || !stat.isFile() || stat.isSymbolicLink()) + throw new Error('Unverified vector preparation directory; artifacts retained'); + } + // Validate the whole directory before removing anything; stream both passes + // so repeated interrupted attempts cannot require an unbounded entry array. + for await (const entry of await fs.opendir(directory)) { + await fs.unlink(path.join(directory, entry.name)); + } +} + +/** Hard links pin the inode independently of source cleanup. Cross-device + * copies are allowed only for initial local-index adoption, never rebases. + * A directly named preparation directory makes an interrupted copy resumable + * without scanning old jobs or overwriting an unverified existing artifact. */ +export async function retainSharedVectorArtifact( + sourceRoot: string, + destinationRoot: string, + generation: PublishedVectorGeneration, + allowCopy = false, +): Promise { + const artifact = generation.artifactIdentity ?? generation.identity; + const source = await assertOwnedVectorGeneration(sourceRoot, artifact); + const sourceIdentity = await completedFile(source); + const root = await embeddingJobDirectory(destinationRoot); + const directory = embeddingGenerationDirectory(root, artifact); + const parent = path.dirname(directory); + try { + await fs.mkdir(parent, { mode: 0o700 }); + await syncEmbeddingJobDirectory(root); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + await embeddingJobDirectory(parent); + const signature = createHash('sha256') + .update( + JSON.stringify([ + artifact, + generation.vectorIdentity, + generation.inputIdentity, + generation.dimensions, + generation.nodes, + generation.chunks, + generation.indexKind, + ]), + ) + .digest('hex'); + + let present = false; + try { + await fs.lstat(directory); + present = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + if (present) { + const file = await assertOwnedVectorGeneration(root, artifact); + const identity = await completedFile(file); + if ( + file === source || + (sourceIdentity.dev !== '0' && + sourceIdentity.ino !== '0' && + sameFile(identity, sourceIdentity)) + ) + return; + const retained = await receipt(directory, signature); + if (!sameFile(retained.file, identity)) + throw new Error('Retained vector file changed; artifacts retained'); + return; + } + + const preparing = path.join(parent, `.retain-${artifact}`); + let intent: RetentionReceipt; + try { + await fs.mkdir(preparing, { mode: 0o700 }); + intent = { version: 1, signature, source: sourceIdentity }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + await syncEmbeddingJobDirectory(parent); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + try { + intent = await receipt(preparing, signature); + } catch (readError) { + if ((readError as NodeJS.ErrnoException).code !== 'ENOENT') throw readError; + await recoverEmptyPreparation(preparing); + intent = { version: 1, signature, source: sourceIdentity }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + await syncEmbeddingJobDirectory(parent); + } + if (!sameFile(intent.source, sourceIdentity)) + throw new Error('Vector retention source changed; artifacts retained'); + } + const target = path.join(preparing, 'vectors'); + if (intent.file) { + if (!sameFile(intent.file, await completedFile(target))) + throw new Error('Prepared vector file changed; artifacts retained'); + } else { + // A verified intent owns only this unacknowledged preparation file. + // Its immutable source remains available even if a copy was interrupted. + try { + await completedFile(target); + await fs.unlink(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + try { + await fs.link(source, target); + } catch (error) { + if (!allowCopy || (error as NodeJS.ErrnoException).code !== 'EXDEV') throw error; + await fs.copyFile(source, target, constants.COPYFILE_EXCL); + const handle = await fs.open(target, 'r+'); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + if (!sameFile(sourceIdentity, await completedFile(source))) + throw new Error('Vector source changed during retention; artifacts retained'); + intent = { ...intent, file: await completedFile(target) }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + } + await fs.rename(preparing, directory); + await syncEmbeddingJobDirectory(parent); +} diff --git a/gitnexus/test/unit/shared-store-generations.test.ts b/gitnexus/test/unit/shared-store-generations.test.ts index 12c8461a5..c332d7c3f 100644 --- a/gitnexus/test/unit/shared-store-generations.test.ts +++ b/gitnexus/test/unit/shared-store-generations.test.ts @@ -5,7 +5,9 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ensurePrivateSharedGraph, + featureKeyOf, publishSharedGraph, + seedSharedSlot, } from '../../src/core/shared-store-analyze.js'; import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import { reclaimSharedStore } from '../../src/storage/shared-store-lifecycle.js'; @@ -33,6 +35,37 @@ describe('shared-store logical generations', () => { parseCacheIdentity: { base: 'fixture-parser' }, ...extra, }); + const modern = (extra: Partial = {}): RepoMeta => + metadata({ + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'c'.repeat(64) }, + ...extra, + }); + const requireMeta = async (directory: string): Promise => { + const meta = await loadMeta(directory); + if (!meta) throw new Error(`Missing fixture metadata: ${directory}`); + return meta; + }; + const vector = (): NonNullable => ({ + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', + }); + const vectorFile = (directory: string) => + path.join(directory, 'embeddings', 'b'.repeat(64), 'vectors'); + const writeVector = async (directory: string) => { + const file = vectorFile(directory); + await fs.mkdir(path.dirname(file), { recursive: true }); + // These fixtures test filesystem ownership; native seals are tested in + // embedding-vector-store.test.ts and the published-job controller suite. + await fs.writeFile(file, 'sealed vector filesystem fixture'); + return file; + }; beforeEach(async () => { root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-shared-generations-'))); @@ -53,6 +86,180 @@ describe('shared-store logical generations', () => { await fs.rm(root, { recursive: true, force: true }); }); + it('shares modern graph facts independently of logical IDs and separate vector work', () => { + const a = modern({ stats: { embeddings: 0 } }); + const b = modern({ + graphGeneration: { version: 1, coveredContent: 'c'.repeat(64), id: 'f'.repeat(32) }, + embeddingDims: 4, + vectorGeneration: vector(), + embeddingJob: { version: 1, identity: '1'.repeat(64), graphGeneration: 'f'.repeat(32) }, + legacyVectorSource: '2'.repeat(64), + stats: { embeddings: 1 }, + }); + expect(featureKeyOf(a)).toBe(featureKeyOf(b)); + expect( + featureKeyOf({ + ...b, + graphGeneration: { version: 1, id: 'f'.repeat(32), coveredContent: '9'.repeat(64) }, + }), + ).not.toBe(featureKeyOf(a)); + expect(featureKeyOf({ ...b, schemaFingerprint: 'different graph schema' })).not.toBe( + featureKeyOf(a), + ); + expect(featureKeyOf(metadata({ stats: { embeddings: 0 } }))).not.toBe( + featureKeyOf(metadata({ stats: { embeddings: 1 } })), + ); + }); + + it('publishes a completed vector pin without copying its contents', async () => { + await fs.writeFile(path.join(slot, generation), 'completed graph'); + const original = await writeVector(slot); + await saveMeta( + slot, + modern({ graphPath: generation, vectorGeneration: vector(), stats: { embeddings: 1 } }), + ); + const copy = vi.spyOn(fs, 'copyFile'); + await publishSharedGraph(layout, repo, head, () => {}); + const committed = path.dirname(resolveGraphPath(slot)); + expect((await loadMeta(committed))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(committed))).ino).toBe((await fs.stat(original)).ino); + expect(copy).not.toHaveBeenCalled(); + }); + + it('seeds independent completed vector pins that survive collection of their source', async () => { + await fs.writeFile(path.join(slot, generation), 'completed graph'); + const original = await writeVector(slot); + const inode = (await fs.stat(original)).ino; + await saveMeta( + slot, + modern({ graphPath: generation, vectorGeneration: vector(), stats: { embeddings: 1 } }), + ); + await publishSharedGraph(layout, repo, head, () => {}); + const sibling = path.join(root, 'sibling'); + git('worktree', 'add', '-q', '-b', 'sibling', sibling); + const next = sharedStoreLayout(path.basename(layout.root), sibling); + await seedSharedSlot(next, sibling, () => {}); + expect((await fs.stat(vectorFile(next.checkoutSlot))).ino).toBe(inode); + const seeded = await requireMeta(next.checkoutSlot); + expect(seeded?.vectorGeneration).toEqual(vector()); + await fs.writeFile(path.join(next.checkoutSlot, generation), 'equivalent private graph'); + await saveMeta(next.checkoutSlot, { ...seeded, graphPath: generation }); + await fs.rm(slot, { recursive: true, force: true }); + expect((await reclaimSharedStore(layout.root)).removed).toHaveLength(1); + expect(await fs.readFile(vectorFile(next.checkoutSlot), 'utf8')).toBe( + 'sealed vector filesystem fixture', + ); + }); + + it('keeps pending job ownership in its checkout when publishing graph facts', async () => { + const job = { version: 1 as const, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }; + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await saveMeta(slot, modern({ graphPath: generation, embeddingJob: job })); + await publishSharedGraph(layout, repo, head, () => {}); + expect((await loadMeta(slot))?.embeddingJob).toEqual(job); + const committed = await loadMeta(path.dirname(resolveGraphPath(slot))); + expect(committed?.graphGeneration).toEqual(modern().graphGeneration); + expect(committed?.embeddingJob).toBeUndefined(); + expect(committed?.legacyVectorSource).toBeUndefined(); + }); + + it('adds completed vectors to an already published graph without replacing that graph', async () => { + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await saveMeta(slot, modern({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + const graph = resolveGraphPath(slot); + const graphInode = (await fs.stat(graph)).ino; + const original = await writeVector(slot); + const before = await requireMeta(slot); + await saveMeta(slot, { ...before, vectorGeneration: vector(), stats: { embeddings: 1 } }); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(graph); + expect((await fs.stat(graph)).ino).toBe(graphInode); + expect((await loadMeta(path.dirname(graph)))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(path.dirname(graph)))).ino).toBe( + (await fs.stat(original)).ino, + ); + }); + + it('adopts a completed local generation with its vectors while retaining the source', async () => { + const source = path.join(repo, '.gitnexus'); + await fs.mkdir(source); + await fs.writeFile(path.join(source, generation), 'local generation'); + const original = await writeVector(source); + await saveMeta( + source, + modern({ + storagePath: source, + graphPath: generation, + vectorGeneration: vector(), + stats: { embeddings: 1 }, + }), + ); + await seedSharedSlot(layout, repo, () => {}); + expect(await fs.readFile(resolveGraphPath(slot), 'utf8')).toBe('local generation'); + expect((await loadMeta(slot))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(slot))).ino).toBe((await fs.stat(original)).ino); + expect(await fs.readFile(path.join(source, generation), 'utf8')).toBe('local generation'); + }); + + it('retains historical coverage and the exact-scan limit in shared metadata', async () => { + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await writeVector(slot); + const completed = { + ...vector(), + graphGeneration: 'f'.repeat(32), + preservedForGraph: 'a'.repeat(32), + chunks: 5, + }; + const meta = modern({ + graphPath: generation, + vectorGeneration: completed, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb', status: 'available' }, + vectorSearch: { provider: 'exact-scan', status: 'exact-scan', exactScanLimit: 2 }, + }, + }); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + const published = await requireMeta(path.dirname(resolveGraphPath(slot))); + expect(published.vectorGeneration).toEqual(completed); + expect(published.capabilities?.vectorSearch).toMatchObject({ + status: 'unavailable', + reason: 'vector-index-pending', + coverage: 'preserved', + sourceGraphGeneration: 'f'.repeat(32), + exactScanLimit: 2, + }); + expect(featureKeyOf(published)).toBe(featureKeyOf(meta)); + }); + + it('keeps a modern private graph when an existing shared graph has no metadata', async () => { + const meta = modern({ graphPath: generation }); + const target = path.join(layout.commitsDir, `${head}-${featureKeyOf(meta)}`); + await fs.mkdir(target, { recursive: true }); + await fs.writeFile(path.join(target, 'lbug'), 'unverified shared graph'); + await fs.writeFile(path.join(slot, generation), 'verified private graph'); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(path.join(slot, generation)); + expect(await fs.readFile(resolveGraphPath(slot), 'utf8')).toBe('verified private graph'); + }); + + it("refuses to strand this checkout's pending local job during adoption", async () => { + const source = path.join(repo, '.gitnexus'); + await fs.mkdir(source); + await fs.writeFile(path.join(source, generation), 'local generation'); + const job = { version: 1 as const, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }; + await saveMeta( + source, + modern({ storagePath: source, graphPath: generation, embeddingJob: job }), + ); + await expect(seedSharedSlot(layout, repo, () => {})).rejects.toThrow('pending embedding job'); + expect((await loadMeta(source))?.embeddingJob).toEqual(job); + expect(await loadMeta(slot)).toBeNull(); + }); + it.each([true, false])( 'keeps an owned generation in place without copying or dropping its pointer (copy=%s)', async (copy) => { diff --git a/gitnexus/test/unit/shared-vector-artifact.test.ts b/gitnexus/test/unit/shared-vector-artifact.test.ts new file mode 100644 index 000000000..014a8772e --- /dev/null +++ b/gitnexus/test/unit/shared-vector-artifact.test.ts @@ -0,0 +1,159 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { retainSharedVectorArtifact } from '../../src/storage/shared-vector-artifact.js'; +import type { PublishedVectorGeneration } from '../../src/storage/vector-generation.js'; + +const generation: PublishedVectorGeneration = { + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + vectorIdentity: 'c'.repeat(64), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', +}; +let root: string, + source: string, + destination: string, + original: string, + retained: string, + preparing: string; +const differentDevice = () => Object.assign(new Error('Different filesystem'), { code: 'EXDEV' }); +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-retain-vectors-')); + source = path.join(root, 'source'); + destination = path.join(root, 'destination'); + original = path.join(source, 'embeddings', generation.identity, 'vectors'); + retained = path.join(destination, 'embeddings', generation.identity, 'vectors'); + preparing = path.join(destination, 'embeddings', `.retain-${generation.identity}`); + await fs.mkdir(path.dirname(original), { recursive: true }); + await fs.mkdir(destination); + // Filesystem ownership fixture; native immutability/identity is covered by + // embedding-vector-store.test.ts and embedding-job-controller.test.ts. + await fs.writeFile(original, 'completed immutable vectors'); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +it('retains an independent hard link without copying or rewriting it on retry', async () => { + const copy = vi.spyOn(fs, 'copyFile'); + await retainSharedVectorArtifact(source, destination, generation); + const inode = (await fs.stat(retained)).ino; + expect(inode).toBe((await fs.stat(original)).ino); + await retainSharedVectorArtifact(source, destination, generation); + expect((await fs.stat(retained)).ino).toBe(inode); + expect(copy).not.toHaveBeenCalled(); + await fs.unlink(original); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('copies only for explicit initial adoption across filesystems, then reuses the receipt', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + await retainSharedVectorArtifact(source, destination, generation, true); + expect((await fs.stat(retained)).ino).not.toBe((await fs.stat(original)).ino); + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledOnce(); + expect(await fs.readFile(retained, 'utf8')).toBe(await fs.readFile(original, 'utf8')); +}); + +it('retains its source and refuses an unadmitted cross-device copy', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toMatchObject({ + code: 'EXDEV', + }); + expect(copy).not.toHaveBeenCalled(); + expect(await fs.readFile(original, 'utf8')).toBe('completed immutable vectors'); + await expect(fs.stat(retained)).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('retries a partial initial copy without overwriting its immutable source', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile').mockImplementationOnce(async (_from, to) => { + await fs.writeFile(to, 'partial'); + throw Object.assign(new Error('Disk full'), { code: 'ENOSPC' }); + }); + await expect( + retainSharedVectorArtifact(source, destination, generation, true), + ).rejects.toMatchObject({ code: 'ENOSPC' }); + expect(await fs.readFile(original, 'utf8')).toBe('completed immutable vectors'); + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledTimes(2); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('publishes an already durable copy after interruption without copying again', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + const rename = fs.rename; + let interrupted = false; + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + if (from === preparing && !interrupted) { + interrupted = true; + throw new Error('Interrupted before publication'); + } + return rename(from, to); + }); + await expect(retainSharedVectorArtifact(source, destination, generation, true)).rejects.toThrow( + 'Interrupted', + ); + const inode = (await fs.stat(path.join(preparing, 'vectors'))).ino; + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledOnce(); + expect((await fs.stat(retained)).ino).toBe(inode); +}); + +it('does not replace an existing artifact without a matching retention receipt', async () => { + await fs.mkdir(path.dirname(retained), { recursive: true }); + await fs.writeFile(retained, 'foreign or pending artifact'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow(); + expect(await fs.readFile(retained, 'utf8')).toBe('foreign or pending artifact'); +}); + +it('rejects an unfinished source without moving its sidecars', async () => { + await fs.writeFile(original + '.wal', 'pending native writes'); + const link = vi.spyOn(fs, 'link'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow( + 'unfinished native writes', + ); + expect(link).not.toHaveBeenCalled(); + expect(await fs.readFile(original + '.wal', 'utf8')).toBe('pending native writes'); +}); + +it('recovers a crash before the first intent record became durable', async () => { + await fs.mkdir(preparing, { recursive: true }); + await fs.writeFile( + path.join(preparing, `.retained.json.${randomUUID()}.tmp`), + 'incomplete record', + ); + await retainSharedVectorArtifact(source, destination, generation); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('retains an unreadable preparation and its files as evidence', async () => { + await fs.mkdir(preparing, { recursive: true }); + await fs.writeFile(path.join(preparing, 'retained.json'), 'corrupt'); + await fs.writeFile(path.join(preparing, 'vectors'), 'unverified work'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow(); + expect(await fs.readFile(path.join(preparing, 'vectors'), 'utf8')).toBe('unverified work'); +}); + +it('retains an unacknowledged preparation with unexpected files', async () => { + await fs.mkdir(preparing, { recursive: true }); + const temporary = path.join(preparing, `.retained.json.${randomUUID()}.tmp`); + await fs.writeFile(temporary, 'incomplete record'); + await fs.writeFile(path.join(preparing, 'vectors'), 'unverified work'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow( + 'artifacts retained', + ); + expect(await fs.readFile(temporary, 'utf8')).toBe('incomplete record'); + expect(await fs.readFile(path.join(preparing, 'vectors'), 'utf8')).toBe('unverified work'); +}); From e9b2b13d6744c014e06d1859755b9c00ed16ddd7 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 13:43:12 +0000 Subject: [PATCH 26/67] feat(analyze): retain caches across phase handoff Keep prepared staging during lock reacquisition and share cache publication between the graph writer and exit continuation. Bound the filesystem handoff to chunk keys; never copy live cache contents. Refs #3598 --- gitnexus/src/core/analysis-parse-cache.ts | 253 ++++++++++++++++++ gitnexus/src/core/run-analyze.ts | 98 +------ gitnexus/src/storage/index-lock.ts | 10 + .../test/unit/analysis-parse-cache.test.ts | 195 ++++++++++++++ .../unit/analysis-staging-retention.test.ts | 75 ++++++ 5 files changed, 544 insertions(+), 87 deletions(-) create mode 100644 gitnexus/src/core/analysis-parse-cache.ts create mode 100644 gitnexus/test/unit/analysis-parse-cache.test.ts create mode 100644 gitnexus/test/unit/analysis-staging-retention.test.ts diff --git a/gitnexus/src/core/analysis-parse-cache.ts b/gitnexus/src/core/analysis-parse-cache.ts new file mode 100644 index 000000000..a6a8cbcd8 --- /dev/null +++ b/gitnexus/src/core/analysis-parse-cache.ts @@ -0,0 +1,253 @@ +/** Parser cache publication shared by an in-process writer and its exit supervisor. */ +import fs from 'node:fs/promises'; +import { constants } from 'node:fs'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import { + loadParseCache, + persistParseCacheChunk, + pruneCache, + saveParseCache, + type ParseCache, +} from '../storage/parse-cache.js'; +import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; +import { withStoreLock } from '../storage/shared-store-lifecycle.js'; +import type { SharedStoreLayout } from '../storage/shared-store.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { listStoreMetaRoots } from './shared-store-analyze.js'; + +interface CachePublicationOptions { + storagePath: string; + metadataDirectory: string; + cacheRoot: string; + sharedStore?: SharedStoreLayout; + collectBranchKeys: ( + root: string, + exclude: string, + ) => Promise<{ keys: Set; complete: boolean }>; + log: (message: string) => void; +} + +/** Called only after graph publication, under the ordinary index lock. */ +export async function publishAnalysisParseCache( + cache: ParseCache, + options: CachePublicationOptions, +): Promise { + const { storagePath, metadataDirectory, cacheRoot, sharedStore, collectBranchKeys, log } = + options; + const publish = async () => { + const listing = sharedStore + ? await listStoreMetaRoots(sharedStore) + : { roots: [storagePath], complete: true }; + const siblingKeys = new Set(); + let complete = listing.complete; + for (const root of listing.roots) { + const folded = await collectBranchKeys(root, metadataDirectory); + for (const key of folded.keys) siblingKeys.add(key); + if (!folded.complete) complete = false; + } + // Re-read inside the publication lock: another checkout can have saved + // since parsing began. Metadata alone never admits an incompatible shard. + const retained = await loadParseCache( + cacheRoot, + cache.identity, + cache.writeStoragePath ?? cacheRoot, + ); + cache.storagePath = cacheRoot; + cache.onDiskKeys = new Set(); + for (const key of retained.onDiskKeys ?? []) { + if ( + !cache.stagedKeys?.has(key) && + !cache.staleKeys?.has(key) && + (cache.usedKeys.has(key) || !complete || siblingKeys.has(key)) + ) + cache.onDiskKeys.add(key); + } + for (const [key, value] of retained.entries) { + if ( + !cache.usedKeys.has(key) && + !cache.stagedKeys?.has(key) && + !cache.staleKeys?.has(key) && + (!complete || siblingKeys.has(key)) + ) + cache.entries.set(key, value); + } + if (complete) { + for (const key of siblingKeys) cache.usedKeys.add(key); + } else { + log('Parse cache: a branch meta was unreadable — retaining all cached chunks (#2106).'); + for (const key of cache.entries.keys()) cache.usedKeys.add(key); + for (const key of cache.onDiskKeys) cache.usedKeys.add(key); + } + const pruned = pruneCache(cache, cache.usedKeys); + if (pruned > 0) log(`Parse cache: pruned ${pruned} stale chunk entries`); + const savedKeys = await saveParseCache(cacheRoot, cache); + await mergeStagedDurableParsedFileStore( + cacheRoot, + cache.writeStoragePath ?? cacheRoot, + cache.identity.base, + new Set(savedKeys), + ); + }; + await (sharedStore ? withStoreLock(sharedStore, 'cache', publish) : publish()); +} + +// This contains only chunk keys, never ASTs, graph data or source content. +// The fixed ceiling fits the phase headroom; failure merely leaves this +// optional cache unpublished, and never changes graph/embedding correctness. +const MAX_HANDOFF_BYTES = 16 * 1024 * 1024; +const KEY = /^[a-f0-9]{64}$/; +interface CacheHandoff { + version: 1; + generation: string; + stage: string; + identity: ParseCache['identity']; + used: string[]; + staged: string[]; + stale: string[]; +} +const checksum = (payload: string) => createHash('sha256').update(payload).digest('hex'); +const receiptName = (generation: string) => { + if (!/^[a-f0-9]{32}$/.test(generation)) throw new Error('Invalid cache publication generation'); + return `analysis-cache-${generation}.json`; +}; + +async function ownedStage(root: string, relative: string): Promise { + if (!/^parse-rebuild\/run\.[A-Za-z0-9]{6}$/.test(relative)) + throw new Error('Invalid prepared parser cache stage'); + let current = await embeddingJobDirectory(root); + for (const part of relative.split('/')) + current = await embeddingJobDirectory(path.join(current, part)); + return current; +} + +/** Flush only memory fallback shards. Existing live shards stay in place; + * the supervisor will re-read their current identity under the cache lock. */ +export async function stageAnalysisParseCache( + root: string, + generation: string, + cache: ParseCache, +): Promise { + root = await embeddingJobDirectory(root); + const name = receiptName(generation); + if (!cache.writeStoragePath) + throw new Error('Prepared parser cache has no owned staging directory'); + const relative = path.relative(root, cache.writeStoragePath).split(path.sep).join('/'); + await ownedStage(root, relative); + // Bound each selection before copying or serializing it. Unknown/malformed + // keys cannot become filesystem paths. + for (const keys of [cache.usedKeys, cache.stagedKeys, cache.staleKeys]) { + if (!keys) continue; + if (keys.size > Math.floor(MAX_HANDOFF_BYTES / 3 / 70)) + throw new Error('Prepared parser cache key set exceeds its bound'); + for (const key of keys) + if (!KEY.test(key)) throw new Error('Prepared parser cache contains an invalid key'); + } + for (const [key, value] of cache.entries) await persistParseCacheChunk(cache, key, value); + if (cache.entries.size) + throw new Error('Could not persist prepared parser cache fallback shards'); + const payload: CacheHandoff = { + version: 1, + generation, + stage: relative, + identity: cache.identity, + used: [...cache.usedKeys], + staged: [...(cache.stagedKeys ?? [])], + stale: [...(cache.staleKeys ?? [])], + }; + const serialized = JSON.stringify(payload); + const bytes = Buffer.from(JSON.stringify({ payload, checksum: checksum(serialized) })); + if (bytes.length > MAX_HANDOFF_BYTES) + throw new Error('Prepared parser cache receipt exceeds its bound'); + const temporary = path.join(root, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(root, name)); + await syncEmbeddingJobDirectory(root); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +/** The published graph binds the generation, parser identity and used keys. + * Missing cache state is optional; malformed state is never used as a cache. */ +export async function loadAnalysisParseCache( + root: string, + metadata: RepoMeta, +): Promise { + const generation = metadata.graphGeneration?.id; + if (!generation) throw new Error('Cache publication requires a published graph generation'); + root = await embeddingJobDirectory(root); + const file = path.join(root, receiptName(generation)); + let stat; + try { + stat = await fs.lstat(file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_HANDOFF_BYTES) + throw new Error('Unverified prepared parser cache receipt'); + const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + let bytes: Buffer; + try { + const opened = await handle.stat(); + if ( + !opened.isFile() || + opened.size !== stat.size || + opened.ino !== stat.ino || + opened.dev !== stat.dev + ) + throw new Error('Prepared parser cache receipt changed'); + bytes = Buffer.alloc(stat.size + 1); + let length = 0; + while (length < bytes.length) { + const result = await handle.read(bytes, length, bytes.length - length, length); + if (!result.bytesRead) break; + length += result.bytesRead; + } + if (length !== stat.size) throw new Error('Prepared parser cache receipt changed'); + bytes = bytes.subarray(0, length); + } finally { + await handle.close(); + } + const envelope = JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(bytes)); + const payload = envelope.payload as CacheHandoff; + const keys = (value: unknown): value is string[] => + Array.isArray(value) && value.every((key) => typeof key === 'string' && KEY.test(key)); + if ( + !payload || + envelope.checksum !== checksum(JSON.stringify(payload)) || + payload.version !== 1 || + payload.generation !== generation || + typeof payload.stage !== 'string' || + !payload.identity || + typeof payload.identity.base !== 'string' || + typeof payload.identity.pdg !== 'string' || + payload.identity.base !== metadata.parseCacheIdentity?.base || + (metadata.parseCacheIdentity.pdg && + payload.identity.pdg !== metadata.parseCacheIdentity.pdg.identity) || + !keys(payload.used) || + !keys(payload.staged) || + !keys(payload.stale) + ) + throw new Error('Prepared parser cache does not match the published graph'); + const usedKeys = new Set(metadata.cacheKeys ?? []); + if (payload.used.length !== usedKeys.size || payload.used.some((key) => !usedKeys.has(key))) + throw new Error('Prepared parser cache selection differs from the published graph'); + return { + identity: payload.identity, + entries: new Map(), + usedKeys, + writeStoragePath: await ownedStage(root, payload.stage), + stagedKeys: new Set(payload.staged), + staleKeys: new Set(payload.stale), + }; +} diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 85eedc469..45a625966 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -208,7 +208,6 @@ import { import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { ensurePrivateSharedGraph, - listStoreMetaRoots, leaveSharedStore, optedInSlotToLeave, registerLeftStore, @@ -216,7 +215,7 @@ import { resolveOptedInStore, seedSharedSlot, } from './shared-store-analyze.js'; -import { withStoreLock } from '../storage/shared-store-lifecycle.js'; +import { publishAnalysisParseCache } from './analysis-parse-cache.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js'; import { DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, @@ -268,14 +267,11 @@ import { } from './parse-cache-identity.js'; import { loadParseCache, - saveParseCache, - pruneCache, createColdParseRebuildDir, getColdParseRebuildDir, emptyParseCache, forgetCreatedParseCacheDir, } from '../storage/parse-cache.js'; -import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js'; import { getCurrentCheckout, getRemoteUrl, @@ -5764,89 +5760,17 @@ async function runFullAnalysisInner( // so the cache file size stays bounded across runs (chunks whose // composition no longer matches anything in the current scan are dead // weight; the parse phase populates `usedKeys` as it processes chunks). - const saveCaches = async (): Promise => { - if (!parseCache) return; - // #2106 R6: the parse cache + durable store are shared across branches. - // Before pruning to this run's keys, fold in the OTHER branches' recorded - // chunk keys so a branch switch doesn't evict their still-live shards. - // Adding to usedKeys makes them survive pruneCache AND land in the saved - // index (saveParseCache builds the index from usedKeys). Excludes this - // run's own meta dir, so a single-branch repo folds in nothing → prune - // set byte-identical to today. A shared store (#3352) folds in every - // member checkout and commit graph the same way. - // An unlistable store directory starts the fold incomplete, so the - // retention branch below keeps other slots' chunks. - const listing = writeTarget.sharedStore - ? await listStoreMetaRoots(writeTarget.sharedStore) - : { roots: [storagePath], complete: true }; - const siblingKeys = new Set(); - let complete = listing.complete; - for (const root of listing.roots) { - const folded = await collectBranchCacheKeys(root, metaDir); - for (const k of folded.keys) siblingKeys.add(k); - if (!folded.complete) complete = false; - } - // Another slot may have published since this run loaded its cache. - // Refresh eligibility inside the publication lock for EVERY run. This - // is also the first persistent read for a bypass run, after parsing. - // Metadata alone cannot admit foreign shards, and live entries cannot - // replace this run's staged repairs or revive its retired chunks. - const retained = await loadParseCache(cacheRoot, parseCache.identity, coldParseRebuildDir); - parseCache.storagePath = cacheRoot; - parseCache.onDiskKeys = new Set(); - for (const key of retained.onDiskKeys ?? []) { - if ( - !parseCache.stagedKeys?.has(key) && - !parseCache.staleKeys?.has(key) && - (parseCache.usedKeys.has(key) || !complete || siblingKeys.has(key)) - ) { - parseCache.onDiskKeys.add(key); - } - } - for (const [key, value] of retained.entries) { - if ( - !parseCache.usedKeys.has(key) && - !parseCache.stagedKeys?.has(key) && - !parseCache.staleKeys?.has(key) && - (!complete || siblingKeys.has(key)) - ) { - parseCache.entries.set(key, value); - } - } - if (complete) { - for (const k of siblingKeys) parseCache.usedKeys.add(k); - } else { - // Fail-safe toward retention: a sibling meta was unreadable, so keep - // everything currently loaded rather than evict on incomplete info. - log('Parse cache: a branch meta was unreadable — retaining all cached chunks (#2106).'); - for (const k of parseCache.entries.keys()) parseCache.usedKeys.add(k); - for (const k of parseCache.onDiskKeys ?? []) parseCache.usedKeys.add(k); - } - const pruned = pruneCache(parseCache, parseCache.usedKeys); - if (pruned > 0) { - log(`Parse cache: pruned ${pruned} stale chunk entries`); - } - const savedKeys = await saveParseCache(cacheRoot, parseCache); - // Prune the durable ParsedFile store to EXACTLY the parse cache's - // surviving keys (#2038 warm-cache coverage), so the two content-addressed - // stores stay coherent: a chunk is "cached" iff both its parse-cache shard - // and its durable shards exist. A retired chunk — worker-quarantined, or - // one whose failed durable reset left an uncleared generation behind - // (#3204) — is filtered out of `savedKeys`, so it drops out of the - // durable index here and re-dispatches next run. Same try/catch — a - // durable-store write must never - // break an otherwise successful run (next run treats it as a miss). - await mergeStagedDurableParsedFileStore( - cacheRoot, - parseCache.writeStoragePath ?? cacheRoot, - parseCache.identity.base, - new Set(savedKeys), - ); - }; try { - await (writeTarget.sharedStore - ? withStoreLock(writeTarget.sharedStore, 'cache', saveCaches) - : saveCaches()); + if (parseCache) { + await publishAnalysisParseCache(parseCache, { + storagePath, + metadataDirectory: metaDir, + cacheRoot, + sharedStore: writeTarget.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + } } catch (e) { log(`Warning: could not save parse cache (${(e as Error).message}); continuing.`); } diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index 5d3b6cc47..486d21d91 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -441,6 +441,16 @@ const deniedCreateHandle = ( * `branches//` sub-slot (which owns its own lock + sweep). Non-recursive. */ export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => void): void => { + // A graph worker releases this lock before its supervisor reacquires it. + // Presence (even malformed or unreadable) retains the attempt until the + // publication coordinator validates it; lock ownership alone cannot prove + // that these stages are orphaned. + try { + lstatSync(path.join(lockDir, 'analysis-publication.json')); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return; + } // Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.`, `lbug.staging..wal`, … // Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`. const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/; diff --git a/gitnexus/test/unit/analysis-parse-cache.test.ts b/gitnexus/test/unit/analysis-parse-cache.test.ts new file mode 100644 index 000000000..1de1aac55 --- /dev/null +++ b/gitnexus/test/unit/analysis-parse-cache.test.ts @@ -0,0 +1,195 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { + loadAnalysisParseCache, + publishAnalysisParseCache, + stageAnalysisParseCache, +} from '../../src/core/analysis-parse-cache.js'; +import { + createColdParseRebuildDir, + emptyParseCache, + loadParseCache, + loadParseCacheChunk, + persistParseCacheChunk, + saveParseCache, + type ParseCache, +} from '../../src/storage/parse-cache.js'; +import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { TEST_PARSE_IDENTITY } from '../helpers/cache-identity.js'; + +const generation = 'e'.repeat(32); +const a = 'a'.repeat(64), + b = 'b'.repeat(64), + c = 'c'.repeat(64), + d = 'd'.repeat(64); +const result = (fileCount: number): ParseWorkerResult => ({ + nodes: [], + relationships: [], + symbols: [], + calls: [], + assignments: [], + routes: [], + fetchCalls: [], + fetchWrapperDefs: [], + decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + toolDefs: [], + ormQueries: [], + constructorBindings: [], + fileScopeBindings: [], + parsedFiles: [], + skippedLanguages: {}, + fileCount, +}); +let root: string, stage: string; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-cache-handoff-'))); + stage = await createColdParseRebuildDir(root); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); +const metadata = (keys: string[]): RepoMeta => ({ + repoPath: root, + lastCommit: '1'.repeat(40), + indexedAt: '2026-11-04T00:00:00Z', + graphGeneration: { version: 1, id: generation, coveredContent: 'f'.repeat(64) }, + parseCacheIdentity: { base: TEST_PARSE_IDENTITY.base }, + cacheKeys: keys, +}); +const options = (keys: string[] = [], complete = true) => ({ + storagePath: root, + metadataDirectory: root, + cacheRoot: root, + collectBranchKeys: async () => ({ keys: new Set(keys), complete }), + log: vi.fn(), +}); +async function live(keys: [string, number][]) { + const cache = emptyParseCache(TEST_PARSE_IDENTITY); + for (const [key, count] of keys) { + cache.usedKeys.add(key); + cache.entries.set(key, [result(count)]); + } + await saveParseCache(root, cache); +} +async function restored(meta: RepoMeta): Promise { + const cache = await loadAnalysisParseCache(root, meta); + if (!cache) throw new Error('Missing handoff fixture'); + return cache; +} +async function mutateReceipt(change: (payload: Record) => void) { + const file = path.join(root, `analysis-cache-${generation}.json`); + const envelope = JSON.parse(await fs.readFile(file, 'utf8')); + change(envelope.payload); + envelope.checksum = createHash('sha256').update(JSON.stringify(envelope.payload)).digest('hex'); + await fs.writeFile(file, JSON.stringify(envelope)); +} + +it('hands off keys without copying live shards, then merges a newer sibling publication', async () => { + await live([ + [a, 1], + [b, 2], + ]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys = new Set([a, b, c]); + cache.staleKeys = new Set([b]); + await persistParseCacheChunk(cache, a, [result(11)]); + await persistParseCacheChunk(cache, c, [result(3)]); + const copy = vi.spyOn(fs, 'copyFile'); + await stageAnalysisParseCache(root, generation, cache); + expect(copy).not.toHaveBeenCalled(); + expect( + (await loadParseCacheChunk(await loadParseCache(root, TEST_PARSE_IDENTITY, stage), a))?.[0] + .fileCount, + ).toBe(1); + expect((await fs.readdir(path.join(stage, 'parse-cache'))).sort()).toEqual([ + `${a}.v8`, + `${c}.v8`, + ]); + // A sibling finishes after the graph child exited, before cache publication. + await live([ + [a, 1], + [b, 2], + [d, 4], + ]); + await publishAnalysisParseCache(await restored(metadata([a, b, c])), options([d])); + const published = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + expect([...(published.onDiskKeys ?? [])].sort()).toEqual([a, c, d]); + expect((await loadParseCacheChunk(published, a))?.[0].fileCount).toBe(11); + expect((await loadParseCacheChunk(published, c))?.[0].fileCount).toBe(3); + expect((await loadParseCacheChunk(published, d))?.[0].fileCount).toBe(4); +}); + +it('flushes memory fallback shards before the child can hand off its cache', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + cache.entries.set(a, [result(123)]); + await stageAnalysisParseCache(root, generation, cache); + expect(cache.entries.size).toBe(0); + const recovered = await restored(metadata([a])); + expect(recovered.entries.size).toBe(0); + expect((await loadParseCacheChunk(recovered, a))?.[0].fileCount).toBe(123); + const receipt = await fs.readFile(path.join(root, `analysis-cache-${generation}.json`), 'utf8'); + expect(receipt).not.toContain('fileCount'); +}); + +it('does not revive the old shard when a staged repair disappears', async () => { + await live([[a, 1]]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys.add(a); + await persistParseCacheChunk(cache, a, [result(2)]); + await stageAnalysisParseCache(root, generation, cache); + await fs.unlink(path.join(stage, 'parse-cache', `${a}.v8`)); + await publishAnalysisParseCache(await restored(metadata([a])), options()); + expect( + await loadParseCacheChunk(await loadParseCache(root, TEST_PARSE_IDENTITY, stage), a), + ).toBeUndefined(); +}); + +it('retains sibling shards when metadata cannot be fully inspected, while dropping stale keys', async () => { + await live([ + [a, 1], + [b, 2], + [c, 3], + ]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys.add(a); + cache.staleKeys = new Set([b]); + await stageAnalysisParseCache(root, generation, cache); + await publishAnalysisParseCache(await restored(metadata([a])), options([], false)); + const published = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + expect([...(published.onDiskKeys ?? [])].sort()).toEqual([a, c]); +}); + +it.each(['generation', 'parser', 'selection', 'path', 'checksum'])( + 'refuses an incompatible or corrupt %s receipt', + async (kind) => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + await stageAnalysisParseCache(root, generation, cache); + if (kind === 'checksum') { + const file = path.join(root, `analysis-cache-${generation}.json`); + const envelope = JSON.parse(await fs.readFile(file, 'utf8')); + envelope.checksum = '0'.repeat(64); + await fs.writeFile(file, JSON.stringify(envelope)); + } else + await mutateReceipt((payload) => { + if (kind === 'generation') payload.generation = '9'.repeat(32); + if (kind === 'parser') payload.identity = { base: 'other', pdg: 'other' }; + if (kind === 'selection') payload.used = [b]; + if (kind === 'path') payload.stage = '../foreign'; + }); + await expect(loadAnalysisParseCache(root, metadata([a]))).rejects.toThrow(); + expect((await fs.stat(stage)).isDirectory()).toBe(true); + }, +); + +it('treats a missing optional cache receipt as absent', async () => { + expect(await loadAnalysisParseCache(root, metadata([]))).toBeUndefined(); +}); diff --git a/gitnexus/test/unit/analysis-staging-retention.test.ts b/gitnexus/test/unit/analysis-staging-retention.test.ts new file mode 100644 index 000000000..56dba2060 --- /dev/null +++ b/gitnexus/test/unit/analysis-staging-retention.test.ts @@ -0,0 +1,75 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import { acquireIndexLock, sweepStagingArtifacts } from '../../src/storage/index-lock.js'; +import { + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +let root: string; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-prepared-sweep-'))); +}); +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); +}); + +it('keeps the prepared graph through ordinary lock reacquisition and then publishes it', async () => { + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + const metadata: RepoMeta = { + repoPath: root, + lastCommit: 'a'.repeat(40), + indexedAt: '2026-11-04T00:00:00Z', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }; + await saveMeta(root, metadata); + const prepared = await prepareAnalysisPublication(root, stage, metadata, 'b'.repeat(64)); + const lock = await acquireIndexLock(root); + try { + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + const published = await publishPreparedAnalysis(root, { + verify: async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }, + saveMetadata: (meta) => saveMeta(root, meta), + }); + expect(published?.graphGeneration?.id).toBe(prepared.id); + } finally { + lock.release(); + } +}); + +it.each(['malformed', 'directory', 'dangling-link'])( + 'retains staging evidence when the prepared receipt is %s', + async (kind) => { + const receipt = path.join(root, 'analysis-publication.json'); + if (kind === 'directory') await fs.mkdir(receipt); + else if (kind === 'dangling-link') await fs.symlink(path.join(root, 'missing'), receipt); + else await fs.writeFile(receipt, '{'); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'unverified but owned work'); + await fs.writeFile(stage + '.wal', 'native recovery evidence'); + sweepStagingArtifacts(root); + expect(await fs.readFile(stage, 'utf8')).toBe('unverified but owned work'); + expect(await fs.readFile(stage + '.wal', 'utf8')).toBe('native recovery evidence'); + }, +); + +it('scopes the retention marker to its own locked branch slot', async () => { + const branch = path.join(root, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await fs.writeFile(path.join(branch, 'analysis-publication.json'), '{'); + const name = `lbug.staging.${randomUUID()}`; + await fs.writeFile(path.join(branch, name), 'branch work'); + await fs.writeFile(path.join(root, name), 'flat orphan'); + sweepStagingArtifacts(root); + sweepStagingArtifacts(branch); + await expect(fs.stat(path.join(root, name))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(path.join(branch, name), 'utf8')).toBe('branch work'); +}); From ceaf4c26074ff772cc5182aa1e75f20769cabaa2 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 14:19:20 +0000 Subject: [PATCH 27/67] fix(analyze): retain owned generation routing Keep prepared and pending work under its original slot lock during shared-store adoption. Preserve private generation filenames during incremental writes without copying the graph. Drain model and native resources only when graph work actually starts, including FTS repair. Leave the warm no-op path unchanged. Validated 158 tests, source build, test typecheck, formatting and lint. Complete graph gates cover 29 symbols in two files and the expected coordinator flow. End-to-end embedding activation and performance acceptance remain pending. --- gitnexus/src/core/run-analyze.ts | 73 ++++++- .../analysis-generation-routing.test.ts | 182 ++++++++++++++++++ 2 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 gitnexus/test/integration/analysis-generation-routing.test.ts diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 45a625966..da443264c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -201,6 +201,7 @@ import { import { isSharedStoreDisabled, resolveGraphPath, + resolveGraphWritePath, resolveSharedStore, storeRootOfCheckoutSlot, type SharedStoreLayout, @@ -1288,6 +1289,35 @@ interface WriteTarget { sharedStore?: SharedStoreLayout; } +/** Adoption must not move the checkout away from its unfinished private work. + * Only inspected during a local-to-shared transition; established shared slots + * keep their existing fast path. Branch jobs belong to the same local root. */ +async function hasRetainedAnalysisWork(root: string): Promise { + const pending = async (slot: string): Promise => { + try { + await fs.lstat(path.join(slot, 'analysis-publication.json')); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const meta = await loadMeta(slot); + return Boolean(meta?.embeddingJob || meta?.embeddingCheckpoint); + }; + if (await pending(root)) return true; + let branches; + try { + branches = await fs.opendir(path.join(root, 'branches')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } + for await (const branch of branches) { + if (branch.isDirectory() && (await pending(path.join(root, 'branches', branch.name)))) + return true; + } + return false; +} + /** * Resolve which storage slot this analyze writes to, including branch * placement (#2106/#2354). Extracted from the top of the pipeline so the lock @@ -1320,15 +1350,29 @@ async function resolveWriteTarget(repoPath: string, options: AnalyzeOptions): Pr ); } const sharingOff = options.noShare || isSharedStoreDisabled(); - const sharedStore = sharingOff + let sharedStore = sharingOff ? undefined : (resolveSharedStore(repoPath) ?? (await resolveOptedInStore(repoPath, options.shareWith))); + const previousStorage = resolveStoragePath(repoPath); + let retainedStorage: string | undefined; + if ( + sharedStore && + path.resolve(previousStorage) !== path.resolve(sharedStore.checkoutSlot) && + !storeRootOfCheckoutSlot(previousStorage) && + (await hasRetainedAnalysisWork(previousStorage)) + ) { + // Re-resolved after ordinary lock acquisition as well: if the previous + // writer completes while we wait, adoption can proceed on the next lock. + retainedStorage = previousStorage; + sharedStore = undefined; + } // A checkout still registered in a store after sharing was turned off // indexes into its own `.gitnexus` again; its slot is left for `clean --gc`. - const leavingStore = - !sharedStore && storeRootOfCheckoutSlot(resolveStoragePath(repoPath)) !== null; + const leavingStore = !sharedStore && storeRootOfCheckoutSlot(previousStorage) !== null; const explicitStorage = - sharedStore?.checkoutSlot ?? (leavingStore ? defaultStoragePath(repoPath) : undefined); + retainedStorage ?? + sharedStore?.checkoutSlot ?? + (leavingStore ? defaultStoragePath(repoPath) : undefined); const storagePath = explicitStorage ? await requireRegisteredStoragePath( { path: repoPath, storagePath: explicitStorage }, @@ -1364,10 +1408,10 @@ async function resolveWriteTarget(repoPath: string, options: AnalyzeOptions): Pr : {}; const paths = getStoragePaths(repoPath, placement.branch, storagePath); const { metaPath } = paths; - // Analyze always writes a store slot's own graph; a recorded `graphPath` - // (an immutable commit graph) only redirects readers. + // Owned generation files stay writable in place. Only an immutable shared + // commit pointer resolves to a fresh slot-local write target. const lbugPath = storeRootOfCheckoutSlot(storagePath) - ? path.join(path.dirname(metaPath), LBUG_DIRECTORY) + ? resolveGraphWritePath(path.dirname(metaPath)) : paths.lbugPath; return { storagePath, @@ -1447,7 +1491,6 @@ export async function runFullAnalysis( } initialiseSearchFTSStemmer(); initialiseSearchFTSCjkSegmentation(); - if (options.resources) await drainAnalysisNativeResources(); const contentRetention = contentRetentionFromEnvironment(); // Scope the degraded-parse log throttle to this run (module-level counter // would otherwise stay saturated on a reused process). @@ -1739,6 +1782,13 @@ async function runFullAnalysisInner( }; const ftsProfile = ftsProfileForContentRetention(contentRetention); const ftsIndexes = getFtsIndexes(ftsProfile); + const admitGraphResources = async (): Promise => { + if (!options.resources) return; + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + }; // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open // (e.g. the embeddings-cache open) falls back to the default until the hint is @@ -1776,7 +1826,7 @@ async function runFullAnalysisInner( } // Later dirty-flag writes spread the in-memory metadata; keep them from // re-recording the pointer this slot just left. - delete loadedMeta?.graphPath; + if (loadedMeta?.graphPath !== path.basename(lbugPath)) delete loadedMeta?.graphPath; }; const loadedMeta = await loadMeta(metaDir); if (loadedMeta?.incrementalInProgress || options.repairFts) await ensurePrivateGraph(); @@ -1904,6 +1954,7 @@ async function runFullAnalysisInner( 'Run `gitnexus analyze` (full) to rebuild from scratch.', ); } + await admitGraphResources(); await ensureWritableStorage(); // P1 R8: park a poisoned live WAL before opening. Staging never parks — // the live index next to an unpublished staging file must replay its WAL. @@ -2903,6 +2954,7 @@ async function runFullAnalysisInner( } } + await admitGraphResources(); await ensureWritableStorage(); // A forced rebuild reads the old graph only to carry embeddings over; with // none to carry, copying the shared graph would be thrown away unread. @@ -2910,7 +2962,7 @@ async function runFullAnalysisInner( resumeEmbeddingCheckpoint || _deriveEmbeddingMode(options, existingMeta?.stats?.embeddings ?? 0).shouldLoadCache; await ensurePrivateGraph(!options.force || forcedRebuildReadsOldGraph); - delete existingMeta?.graphPath; + if (existingMeta?.graphPath !== path.basename(lbugPath)) delete existingMeta?.graphPath; // Checkpoint 1, final read: a graph copy that just failed forces the rebuild // before the embedding plan, the parse cache, and the emit mode are decided. applyCollectedForce(); @@ -5374,6 +5426,7 @@ async function runFullAnalysisInner( const meta: RepoMeta = { repoPath, storagePath, + ...(path.basename(lbugPath) !== LBUG_DIRECTORY ? { graphPath: path.basename(lbugPath) } : {}), lastCommit: currentCommit, indexedAt: new Date().toISOString(), contentRetention, diff --git a/gitnexus/test/integration/analysis-generation-routing.test.ts b/gitnexus/test/integration/analysis-generation-routing.test.ts new file mode 100644 index 000000000..c302a7742 --- /dev/null +++ b/gitnexus/test/integration/analysis-generation-routing.test.ts @@ -0,0 +1,182 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { ensurePrivateSharedGraph } from '../../src/core/shared-store-analyze.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import * as nativeResources from '../../src/core/lbug/lbug-adapter.js'; +import * as modelResources from '../../src/core/embeddings/embedding-sidecar-client.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { resolveSharedStore } from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +let root: string; +let repo: string; +const generation = `lbug.generation.${'a'.repeat(32)}`; +const options = { skipFts: true, skipAgentsMd: true, skipSkills: true }; +const git = (...args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8', stdio: 'pipe' }).trim(); +const metadata = (extra: Partial = {}): RepoMeta => ({ + repoPath: repo, + lastCommit: git('rev-parse', 'HEAD'), + indexedAt: '2026-11-04T00:00:00Z', + branch: 'main', + ...extra, +}); + +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-generation-routing-'))); + repo = path.join(root, 'repo'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + git('init', '-q', '-b', 'main'); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 1; }\n'); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); +}); +afterEach(async () => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +it.each([false, true])( + 'retains the owned graph filename on a tiny edit (shared=%s)', + async (shared) => { + if (shared) { + git('worktree', 'add', '-q', '-b', 'sibling', path.join(root, 'sibling')); + delete process.env.GITNEXUS_SHARED_STORE; + } + const first = await runFullAnalysis(repo, options, { onProgress: () => {} }); + const slot = first.storagePath; + if (shared) expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + const prior = await loadMeta(slot); + if (!prior) throw new Error('Fixture index is missing'); + const owned = path.join(slot, generation); + await fs.rename(path.join(slot, 'lbug'), owned); + await saveMeta(slot, { ...prior, graphPath: generation }); + const inode = (await fs.stat(owned)).ino; + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 2; }\n'); + const copy = vi.spyOn(fs, 'copyFile'); + const result = await runFullAnalysis(repo, options, { onProgress: () => {} }); + expect(result.alreadyUpToDate).not.toBe(true); + expect((await loadMeta(slot))?.graphPath).toBe(generation); + expect(getStoragePaths(repo, undefined, slot).lbugPath).toBe(owned); + expect((await fs.stat(owned)).ino).toBe(inode); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect( + copy.mock.calls.filter((args) => + args.some((file) => path.basename(String(file)).startsWith('lbug')), + ), + ).toHaveLength(0); + const source = await EmbeddingGraphSource.open(owned, 256 * 1024 * 1024, 1); + try { + const ids: string[] = []; + for await (const key of source.scan('Function')) ids.push(key.id); + expect(ids).toHaveLength(1); + expect((await source.fetch('Function', ids))[0].content).toContain('return 2'); + } finally { + await source.close(); + } + }, + 120_000, +); + +it.each(['pending', 'prepared', 'branch-pending'] as const)( + 'takes the original slot lock before shared adoption with %s work', + async (kind) => { + const local = path.join(repo, '.gitnexus'); + await saveMeta(local, metadata()); + let lockedSlot = local; + if (kind === 'prepared') { + // Even unreadable publication evidence must not be stranded by routing. + await fs.writeFile(path.join(local, 'analysis-publication.json'), '{'); + } else { + if (kind === 'branch-pending') { + git('checkout', '-q', '-b', 'feature'); + lockedSlot = path.dirname(getStoragePaths(repo, 'feature', local).metaPath); + } + await saveMeta( + lockedSlot, + metadata({ + branch: kind === 'branch-pending' ? 'feature' : 'main', + embeddingJob: { version: 1, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }, + }), + ); + } + git('worktree', 'add', '-q', '-b', 'sibling', path.join(root, 'sibling')); + delete process.env.GITNEXUS_SHARED_STORE; + const shared = resolveSharedStore(repo); + if (!shared) throw new Error('Fixture store is missing'); + const lock = await acquireIndexLock(lockedSlot, { sweep: false }); + const waitedForOwner = new Error('Observed the original owner lock'); + try { + await expect( + runFullAnalysis( + repo, + { + ...options, + ...(kind === 'branch-pending' ? { branch: 'feature' } : {}), + }, + { + onProgress: (phase) => { + if (phase === 'lock') throw waitedForOwner; + throw new Error(`Analysis bypassed its retained work: ${phase}`); + }, + }, + ), + ).rejects.toBe(waitedForOwner); + expect(await loadMeta(shared.checkoutSlot)).toBeNull(); + if (kind !== 'prepared') expect((await loadMeta(lockedSlot))?.embeddingJob).toBeDefined(); + } finally { + lock.release(); + } + }, + 30_000, +); + +it('leaves model resources alone on a no-op, then drains them before actual graph work', async () => { + await runFullAnalysis(repo, options, { onProgress: () => {} }); + const events: string[] = []; + const native = vi + .spyOn(nativeResources, 'drainAnalysisNativeResources') + .mockImplementation(async () => { + events.push('native'); + }); + const model = vi + .spyOn(modelResources, 'drainEmbeddingSidecarForAnalysis') + .mockImplementation(async () => { + events.push('model'); + }); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, + }); + const noop = await runFullAnalysis(repo, { ...options, resources }, { onProgress: () => {} }); + expect(noop.alreadyUpToDate).toBe(true); + expect(native).not.toHaveBeenCalled(); + expect(model).not.toHaveBeenCalled(); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 3; }\n'); + await runFullAnalysis( + repo, + { ...options, resources }, + { + onProgress: (phase) => { + if (phase === 'extracting') { + expect(events.slice(0, 2)).toEqual(['model', 'native']); + events.push('parse'); + } + }, + }, + ); + expect(events).toContain('parse'); +}, 120_000); From a66f2dd1d21c1f30d0353b29edc447493c3fb0ca Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 4 Nov 2026 21:23:14 +0000 Subject: [PATCH 28/67] feat(analyze): wire resumable phase execution Publish verified graphs before embedding and hand off resource ownership across CLI, watch, auto-sync, and HTTP phases. Keep completed vectors readable while durable embedding units resume in a separate generation. Preserve legacy vector reuse, avoid graph rebuilds for dimension changes, and cover interruption boundaries with fresh-process restart tests. Refs #3598 --- RUNBOOK.md | 20 +- gitnexus/README.md | 15 + gitnexus/bench/analyze-resume/README.md | 11 +- gitnexus/bench/analyze-resume/measure.mjs | 44 +- gitnexus/src/cli/analyze-options.ts | 2 + gitnexus/src/cli/analyze-phase-ipc.ts | 83 +++ gitnexus/src/cli/analyze-watch.ts | 354 +++++---- gitnexus/src/cli/analyze.ts | 332 +++++++-- gitnexus/src/cli/embeddings-sync.ts | 172 +---- gitnexus/src/cli/index.ts | 4 + gitnexus/src/core/analysis-embedding-phase.ts | 264 +++++++ .../src/core/analysis-graph-verification.ts | 70 ++ gitnexus/src/core/analysis-host-resources.ts | 81 ++ gitnexus/src/core/analysis-parse-cache.ts | 13 +- .../src/core/analysis-phase-supervisor.ts | 5 +- gitnexus/src/core/analysis-resource-budget.ts | 21 +- gitnexus/src/core/analysis-supervision.ts | 7 + gitnexus/src/core/augmentation/engine.ts | 6 +- .../core/auto-sync/analysis-worker-launch.ts | 214 ++++-- .../core/embeddings/embedding-job-runner.ts | 9 +- .../src/core/embeddings/embedding-pipeline.ts | 25 +- .../embeddings/vector-generation-search.ts | 2 +- gitnexus/src/core/lbug/lbug-adapter.ts | 6 +- .../src/core/lbug/legacy-vector-export.ts | 177 +++++ gitnexus/src/core/lbug/vector-store.ts | 40 +- gitnexus/src/core/run-analyze.ts | 692 ++++++++++++++++-- gitnexus/src/core/shared-store-analyze.ts | 3 + gitnexus/src/server/analyze-job.ts | 71 +- gitnexus/src/server/analyze-launch.ts | 171 +++-- gitnexus/src/server/analyze-worker-core.ts | 65 +- .../src/server/analyze-worker-protocol.ts | 27 +- gitnexus/src/server/analyze-worker.ts | 39 +- gitnexus/src/server/api.ts | 395 +++------- gitnexus/src/storage/analysis-slot-lock.ts | 29 + gitnexus/src/storage/index-capabilities.ts | 2 + .../src/storage/shared-store-lifecycle.ts | 35 +- gitnexus/src/storage/vector-generation.ts | 11 + .../test/helpers/analysis-http-server.mjs | 11 + .../test/helpers/embedding-restart-runner.mjs | 110 +++ .../test/helpers/prepared-analysis-runner.mjs | 48 ++ .../analysis-embedding-flow.test.ts | 176 +++++ .../analysis-publication-flow.test.ts | 496 +++++++++++++ .../analyze-cli-embedding-phases.test.ts | 184 +++++ .../analyze-http-embedding-phases.test.ts | 153 ++++ .../analyze-staged-embedding-recovery.test.ts | 122 ++- .../integration/embedding-job-restart.test.ts | 157 ++++ .../embedding-vector-store.test.ts | 63 ++ .../integration/lbug-readonly-init.test.ts | 29 + .../integration/legacy-vector-export.test.ts | 262 +++++++ .../unit/analysis-embedding-phase.test.ts | 133 ++++ .../test/unit/analysis-host-resources.test.ts | 101 +++ .../test/unit/analysis-parse-cache.test.ts | 29 + .../unit/analysis-resource-budget.test.ts | 28 + gitnexus/test/unit/analyze-api.test.ts | 106 +-- .../analyze-embedding-endpoint-flags.test.ts | 7 +- .../unit/analyze-embeddings-limit.test.ts | 7 +- .../analyze-finalize-failure-exits.test.ts | 7 +- gitnexus/test/unit/analyze-gitnexusrc.test.ts | 2 + .../test/unit/analyze-heap-respawn.test.ts | 93 ++- .../unit/analyze-http-endpoint-error.test.ts | 7 +- gitnexus/test/unit/analyze-job.test.ts | 59 +- .../unit/analyze-launch-branch-settle.test.ts | 70 +- .../test/unit/analyze-launch-collapse.test.ts | 75 ++ .../analyze-lbug-checkpoint-threshold.test.ts | 7 +- .../analyze-local-embedding-error.test.ts | 7 +- .../test/unit/analyze-no-stats-bridge.test.ts | 7 +- .../test/unit/analyze-pagesize-error.test.ts | 7 +- .../test/unit/analyze-phase-handoff.test.ts | 93 +++ .../test/unit/analyze-process-budget.test.ts | 7 +- .../analyze-respawn-progress-terminal.test.ts | 7 +- .../unit/analyze-self-commit-bridge.test.ts | 7 +- .../analyze-undeclared-pair-error.test.ts | 7 +- gitnexus/test/unit/analyze-wal-error.test.ts | 7 +- gitnexus/test/unit/analyze-wipe-error.test.ts | 7 +- .../unit/analyze-worker-pool-size.test.ts | 7 +- .../test/unit/analyze-worker-timeout.test.ts | 7 +- gitnexus/test/unit/api-fts-mode.test.ts | 215 +++++- .../test/unit/api-readonly-wiring.test.ts | 29 +- .../test/unit/augmentation-generation.test.ts | 52 ++ .../unit/auto-sync-analysis-worker.test.ts | 94 ++- .../test/unit/embeddings-sync-command.test.ts | 181 ++--- gitnexus/test/unit/lbug-checkpoint.test.ts | 2 +- .../test/unit/lbug-resource-release.test.ts | 16 +- gitnexus/test/unit/rate-limit.test.ts | 8 +- .../unit/shared-store-generations.test.ts | 86 ++- gitnexus/test/unit/vector-generation.test.ts | 18 + gitnexus/test/unit/watch-cancellation.test.ts | 80 ++ gitnexus/test/unit/watch-paths.test.ts | 28 + 88 files changed, 5742 insertions(+), 1296 deletions(-) create mode 100644 gitnexus/src/core/analysis-embedding-phase.ts create mode 100644 gitnexus/src/core/analysis-graph-verification.ts create mode 100644 gitnexus/src/core/analysis-host-resources.ts create mode 100644 gitnexus/src/core/analysis-supervision.ts create mode 100644 gitnexus/src/core/lbug/legacy-vector-export.ts create mode 100644 gitnexus/src/storage/analysis-slot-lock.ts create mode 100644 gitnexus/test/helpers/analysis-http-server.mjs create mode 100644 gitnexus/test/helpers/embedding-restart-runner.mjs create mode 100644 gitnexus/test/helpers/prepared-analysis-runner.mjs create mode 100644 gitnexus/test/integration/analysis-embedding-flow.test.ts create mode 100644 gitnexus/test/integration/analysis-publication-flow.test.ts create mode 100644 gitnexus/test/integration/analyze-cli-embedding-phases.test.ts create mode 100644 gitnexus/test/integration/analyze-http-embedding-phases.test.ts create mode 100644 gitnexus/test/integration/embedding-job-restart.test.ts create mode 100644 gitnexus/test/integration/legacy-vector-export.test.ts create mode 100644 gitnexus/test/unit/analysis-embedding-phase.test.ts create mode 100644 gitnexus/test/unit/analysis-host-resources.test.ts create mode 100644 gitnexus/test/unit/augmentation-generation.test.ts create mode 100644 gitnexus/test/unit/watch-cancellation.test.ts diff --git a/RUNBOOK.md b/RUNBOOK.md index d65aeb4c4..0b9d18e47 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -129,13 +129,15 @@ costs; compare complete-source candidates separately from older capped inputs. **Important:** If you already had embeddings, a plain `npx gitnexus analyze` **preserves** them (Non-negotiable 5 in [GUARDRAILS.md](GUARDRAILS.md)) — pass `--embeddings` when you also want vectors generated for new or changed nodes, and `--drop-embeddings` only for a deliberate wipe. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none) — but that figure isn't always freshly measured: if a run's embedding-count query can't answer, it carries the previous run's number forward instead of writing a wrong zero. For a certified read, check `capabilities.vectorSearch.status` instead — it reads `unavailable` (never a stale count) whenever GitNexus can't vouch for the live vector index. -**Partial embedding index (analyze exits 0, but some nodes never got embedded):** A long run against a flaky embedding endpoint can finish successfully while a bounded number of sub-batches still fail. Affected nodes are dropped to zero rows (never left half-written) and recorded as a pending `embeddingCheckpoint`; `npx gitnexus status` then reports `incompleteReasons: ["embedding-checkpoint-pending"]`. Recovery is a plain: +**Embedding work interrupted after graph publication:** The graph and keyword index become available before embedding work starts. An embedding failure makes analyze exit nonzero while keeping that graph available and retaining acknowledged embedding work. `npx gitnexus status --json` reports graph freshness separately from embedding progress and vector readiness. Resume with: ```bash npx gitnexus analyze ``` -No `--embeddings` flag needed — a retained checkpoint forces embedding generation for the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. +No `--embeddings` flag is needed for a retained job. `npx gitnexus embeddings sync` also resumes work against an already published graph. Compatible acknowledged units do not call the provider again; a unit interrupted before acknowledgement may replay. A source or embedding-configuration change revalidates reuse, and changing worker counts or memory budgets does not invalidate paid vectors. `--drop-embeddings` deliberately abandons retained vectors. `--force` rebuilds the graph while retaining compatible paid work for reuse. + +Older indexes may instead carry an `embeddingCheckpoint` and report `embedding-checkpoint-pending`. Plain analyze recovers that checkpoint, including any explicitly referenced staging data, before migrating it. Historical vectors without verified provider/model provenance are retained without being relabeled as output from the current model. **Collapsed graph write (analyze exits NON-ZERO and says INCOMPLETE):** A run can finish writing metadata while only a fraction of the relationships it produced are readable back from the index — edges collapsing to a small share of what was built, or a `CodeRelation` table that never materialized (which reads as a persisted count of zero). Because the metadata IS written and the DB does hold rows, nothing looks broken: queries answer with missing edges rather than an error, which is a confident empty answer rather than a failure. `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`, the analyze summary prints `Repository indexed INCOMPLETELY` with the expected and persisted counts, and the CLI exits non-zero so automation is not told an unusable index is fine. @@ -280,6 +282,20 @@ on a partial result. Analyze re-execs Node with a **large old-space heap** when needed (`analyze.ts`). If you still OOM on huge repos, close other processes, avoid `--embeddings` for a first pass, or analyze a smaller path if supported by your workflow. +`--resource-budget ` admits the analyzer process family within one memory +reservation: the frontend, main heap, parser workers, native graph/vector pools, +embedding model, bounded input queue, and headroom. For example: + +```bash +npx gitnexus analyze --embeddings --resource-budget 8192 --memory-budget 3000 --workers 2 +``` + +`--memory-budget` still controls only the main V8 heap. Explicit settings that do +not fit the family budget fail before work starts. Without a family override, +GitNexus derives reservations from available memory and CPUs. The budget controls +admission and queue sizes; it is not an operating-system RSS limit. Watch mode +accepts the same family budget and drains active work before shutdown. + `Unable to allocate memory` / `The buffer pool is full` during graph writes refers to LadybugDB's native buffer pool. Increasing `NODE_OPTIONS`, `--memory-budget`, or worker heap limits does not resize that pool. Recognized diff --git a/gitnexus/README.md b/gitnexus/README.md index d75c91a3c..3b683d431 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -299,6 +299,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and any --max-old-space-size pin +gitnexus analyze --resource-budget 8192 # Process-family memory reservation in MiB, including workers, native pools and embedding work gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) gitnexus auto-sync [init|start|restart|stop|status|reset] # Scheduled remote clone/pull + analyze from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos @@ -842,6 +843,20 @@ To set the main-thread heap yourself on a memory-constrained host, pass the auto-sizer and any `--max-old-space-size` pin. It sizes the main thread only; parse workers keep their own caps. +Use `--resource-budget ` to admit the complete analyzer process family, +including the frontend, main heap, workers, native pools, embedding model, input +queue, and headroom. An explicit heap or pool that cannot fit is rejected before +work starts. Defaults derive from available memory and CPUs; this reservation is +not an operating-system RSS limit. `analyze --watch` accepts the same budget. + +Embedding-enabled analysis publishes the graph and keyword index first. If the +provider fails, analyze exits nonzero while those queries remain available. Run +plain `gitnexus analyze` or `gitnexus embeddings sync` to resume a retained job. +Compatible acknowledged work is reused without provider requests; an interrupted +unacknowledged unit may replay. Changing resource settings does not invalidate +paid vectors. `gitnexus status --json` exposes graph freshness and embedding +progress separately; semantic search uses a completed, validated vector artifact. + Escape hatches (`GITNEXUS_MEMORY=off` to decline the autopilot, `GITNEXUS_WORKER_HEAP_MB` to size workers yourself) are listed in the environment-variable table below — diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 8618ce6d1..90f10a3f2 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -41,10 +41,13 @@ resolved dependency digest from its analyzer receipts. A checkout on a remote filesystem compared with a local checkout is not useful regression evidence. It includes CLI startup, graph writes, embeddings and final index work. Scenarios cover graph-only, warm no-op, one-function incremental edits, fresh/cached HTTP -embeddings and resume after a deterministic provider outage. The outage ends +embeddings, local-model fresh/cached runs, and resume after a deterministic provider outage. The outage ends normally through application error handling; it does not kill native work. `--help` lists corpus, heap, worker and scenario options. Both sides receive the -same explicit allocations; default-budget behavior needs a separate comparison. +same explicit allocations. Run a separate comparison with `--default-resources` +to omit heap, worker and native-pool overrides on both sides. Local scenarios +use the installed default model; download it before timing to keep network +transfer out of the comparison. Raw samples include command/log paths, analyzer receipts, graph/vector counts, provider request/input counts, deterministic input fingerprints and sampled Linux @@ -58,9 +61,9 @@ exits naturally; the harness does not forward a termination signal to native wor Results use paired log-ratio bootstrap intervals and report regression, inconclusive, or no detected regression. Fewer than seven pairs is always smoke-only. The report always marks overall acceptance incomplete: synthetic -counts/input parity do not establish query ranking, local-model performance, +counts/input parity do not establish query ranking, concurrent query latency, exact peak RSS, queue/native reservations, CPU/disk costs, -default-budget behavior or Windows correctness. Those remain part of the final +unselected scenarios/resource profiles or Windows correctness. Those remain part of the final regression/performance matrix. Do not run measurements alongside indexing/tests. Check the harness itself with: diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs index d96745c77..6f458a628 100644 --- a/gitnexus/bench/analyze-resume/measure.mjs +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -10,7 +10,16 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; -const SCENARIOS = ['graph-only', 'noop', 'incremental', 'http-fresh', 'http-cached', 'http-resume']; +const SCENARIOS = [ + 'graph-only', + 'noop', + 'incremental', + 'http-fresh', + 'http-cached', + 'http-resume', + 'local-fresh', + 'local-cached', +]; const sha = (text) => createHash('sha256').update(text).digest('hex'); async function runtimeEnvironment(cli) { @@ -65,7 +74,7 @@ export function assertCompletedCase(scenario, metadata, beforeMetadata) { assert(beforeMetadata?.indexedAt, 'No-op setup lacks a freshness receipt'); assert.equal(metadata.indexedAt, beforeMetadata.indexedAt, 'No-op changed graph freshness'); } - if (scenario.startsWith('http-')) { + if (scenario.startsWith('http-') || scenario.startsWith('local-')) { assert(metadata.stats.embeddings > 0, 'Embedding scenario produced no vectors'); assert( !metadata.embeddingCheckpoint && !metadata.embeddingJob, @@ -321,6 +330,7 @@ async function runCase({ functions, heapMb, workers, + defaultResources, provider, sampleMs, stopRequested, @@ -336,11 +346,12 @@ async function runCase({ GITNEXUS_HOME: path.join(directory, 'home'), GITNEXUS_STORAGE_PATH: store, GITNEXUS_SHARED_STORE: 'off', - GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(256 * 1024 * 1024), GITNEXUS_WORKER_READY_TIMEOUT_MS: '30000', }); - const embedding = scenario.startsWith('http-'); - if (embedding) + if (!defaultResources) env.GITNEXUS_LBUG_BUFFER_POOL_SIZE = String(256 * 1024 * 1024); + const http = scenario.startsWith('http-'); + const embedding = http || scenario.startsWith('local-'); + if (http) Object.assign(env, { GITNEXUS_EMBEDDING_URL: provider.url, GITNEXUS_EMBEDDING_MODEL: 'gitnexus-benchmark', @@ -349,16 +360,8 @@ async function runCase({ GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '16', }); - const args = [ - cli, - 'analyze', - repo, - '--index-only', - '--memory-budget', - String(heapMb), - '--workers', - String(workers), - ]; + const args = [cli, 'analyze', repo, '--index-only']; + if (!defaultResources) args.push('--memory-budget', String(heapMb), '--workers', String(workers)); if (embedding) args.push('--embeddings', '0'); const counters = provider.begin( scenario === 'http-resume' ? Math.max(16, Math.floor((modules * functions) / 3)) : Infinity, @@ -381,7 +384,7 @@ async function runCase({ return result; }; let setup; - if (['noop', 'incremental', 'http-cached', 'http-resume'].includes(scenario)) { + if (['noop', 'incremental', 'http-cached', 'http-resume', 'local-cached'].includes(scenario)) { setup = await invoke('setup', scenario === 'http-resume'); if (scenario === 'http-resume') { assert( @@ -453,6 +456,7 @@ export async function main(argv = process.argv.slice(2)) { functions: { type: 'string', default: '16' }, 'heap-mb': { type: 'string', default: '2048' }, workers: { type: 'string', default: '2' }, + 'default-resources': { type: 'boolean', default: false }, 'sample-ms': { type: 'string', default: '25' }, 'keep-stores': { type: 'boolean', default: false }, help: { type: 'boolean' }, @@ -464,11 +468,13 @@ export async function main(argv = process.argv.slice(2)) { --scenarios LIST ${SCENARIOS.join(',')} --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 + --default-resources Omit heap, worker and native-pool overrides on both sides --keep-stores Retain owned index/home artifacts (logs and measurements always retained) Includes full CLI startup, graph/manifest/native writes and final index work. HTTP resume uses a deterministic provider outage, not an unsafe native-process kill. RSS is sampled Linux process-family RSS; other platforms report it unavailable. -Local-model, live-query latency, default-budget and platform cases remain separate acceptance requirements.`); +Local scenarios use the installed default model; prewarm its download before timing. +Live-query latency and platform cases remain separate acceptance requirements.`); return; } assert(v.baseline && v.output, '--baseline and --output are required'); @@ -531,6 +537,7 @@ Local-model, live-query latency, default-budget and platform cases remain separa availableParallelism: os.availableParallelism(), totalMemory: os.totalmem(), settings: numbers, + resourceProfile: v['default-resources'] ? 'default' : 'explicit-heap-workers-pool', runtime, clis: {}, }, @@ -538,7 +545,7 @@ Local-model, live-query latency, default-budget and platform cases remain separa 'Synthetic corpus only; stats and deterministic input coverage are checked, not query ranking.', 'Linux RSS sampling is a lower-bound observation, not a hard memory limit or exact peak.', 'CPU/disk, phase timings, native reservations and queue peaks are not collected by this harness.', - 'Local model, live query latency, default budgets, Windows and final production integration require additional validation.', + 'Only the selected scenarios and resource profile were measured; live query latency and Windows require additional validation.', ], acceptance: 'incomplete', samples: [], @@ -583,6 +590,7 @@ Local-model, live-query latency, default-budget and platform cases remain separa functions: numbers.functions, heapMb: numbers['heap-mb'], workers: numbers.workers, + defaultResources: v['default-resources'], sampleMs: numbers['sample-ms'], provider, stopRequested: () => interrupted, diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index 2083487e1..79e3c753f 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -127,6 +127,8 @@ export interface AnalyzeOptions { * Integer, minimum 200; CLI-only (not a `.gitnexusrc` key). */ memoryBudget?: string; + /** Total process-family reservation, separate from the main V8 heap flag. */ + resourceBudget?: string; /** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */ workers?: string; /** Process-detection process cap. Positive integer string; `0` is invalid. */ diff --git a/gitnexus/src/cli/analyze-phase-ipc.ts b/gitnexus/src/cli/analyze-phase-ipc.ts index aa7a6c540..f81901134 100644 --- a/gitnexus/src/cli/analyze-phase-ipc.ts +++ b/gitnexus/src/cli/analyze-phase-ipc.ts @@ -25,6 +25,89 @@ export interface CliPreparedAnalysisMessage { selfCommitSafety?: { agentsMd: boolean; claudeMd: boolean }; } +export interface CliAnalysisContinuation { + type: 'continue-analysis'; + writerPid: number; + prepared: CliPreparedAnalysisMessage; +} + +/** Validate both directions; never retain extra fields from an IPC object. */ +export function parseCliPreparedAnalysis(value: unknown): CliPreparedAnalysisMessage { + const input = value as CliPreparedAnalysisMessage; + if (!input || input.type !== 'graph-prepared' || !input.result) + throw new Error('Invalid CLI graph phase message'); + const safety = input.selfCommitSafety; + if (safety && (typeof safety.agentsMd !== 'boolean' || typeof safety.claudeMd !== 'boolean')) + throw new Error('Invalid CLI self-commit snapshot'); + return projectCliPreparedAnalysis( + { ...input.result, preparedGraph: input.handoff, rebuildReasons: [] }, + input.handoff.repoPath, + input.startedAt, + safety + ? new Map([ + ['AGENTS.md', safety.agentsMd], + ['CLAUDE.md', safety.claudeMd], + ]) + : undefined, + ); +} + +export async function receiveCliAnalysisContinuation( + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!process.connected || !process.send) + throw new Error('Analysis continuation requires its connected supervisor'); + return new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + process.removeListener('message', onMessage); + process.removeListener('disconnect', onDisconnect); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); + reject(signal?.reason ?? new Error('Analysis cancelled')); + }; + const onDisconnect = () => { + cleanup(); + reject(new Error('Analysis supervisor disconnected')); + }; + const onMessage = (message: unknown) => { + cleanup(); + try { + const input = message as CliAnalysisContinuation; + if ( + input?.type !== 'continue-analysis' || + !Number.isSafeInteger(input.writerPid) || + input.writerPid < 1 + ) + throw new Error('Invalid CLI continuation'); + resolve({ + type: 'continue-analysis', + writerPid: input.writerPid, + prepared: parseCliPreparedAnalysis(input.prepared), + }); + } catch (error) { + reject(error); + } + }; + const timer = setTimeout(() => { + cleanup(); + reject(new Error('Analysis supervisor did not send its continuation')); + }, 30_000); + process.once('message', onMessage); + process.once('disconnect', onDisconnect); + signal?.addEventListener('abort', onAbort, { once: true }); + process.send!({ type: 'analysis-continuation-ready' }, (error) => { + if (error) { + cleanup(); + reject(error); + } + }); + }); +} + export function projectCliPreparedAnalysis( result: AnalyzeResult, repoPath: string, diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index a5f19d4e1..bb1e75c3c 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -1,6 +1,7 @@ /** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */ import path from 'node:path'; import fs from 'node:fs/promises'; +import { getHeapStatistics } from 'node:v8'; import { watchFile, unwatchFile } from 'node:fs'; import { watch, type FSWatcher } from 'chokidar'; import { createWatchIgnorePredicate } from '../config/ignore-service.js'; @@ -26,6 +27,7 @@ import type { AnalyzeOptions } from './analyze-options.js'; import { ensureHeap } from './analyze.js'; import { cliError, cliInfo, cliWarn } from './cli-message.js'; import { parseIntegerOption } from './int-option.js'; +import { analysisMayEmbed, resolveHostAnalysisResources } from '../core/analysis-host-resources.js'; import { formatInvalidProcessDetectionOverride, parseProcessDetectionBudgetStrings, @@ -188,6 +190,12 @@ export async function resolveWatchOptions( cliWarn(formatInvalidProcessDetectionOverride(flag, raw)); }, ); + const resources = resolveHostAnalysisResources({ + totalMb: positiveInteger(merged.resourceBudget, '--resource-budget'), + mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + workers: workerPoolSize, + includeEmbeddings: await analysisMayEmbed(repoPath, { branch }), + }); setEnvironment( 'GITNEXUS_MAX_FILE_SIZE', @@ -201,6 +209,8 @@ export async function resolveWatchOptions( setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); return { + resources, + graphPublication: 'auto', pdg: merged.pdg, skipFts: merged.skipFts, branch, @@ -453,182 +463,212 @@ export async function watchCommandWithRunnerIdentity( inputPath?: string, cliOptions: WatchCliOptions = {}, ): Promise { - if ( - await ensureHeap({ cleanForwardedTermination: true, memoryBudget: cliOptions.memoryBudget }) - ) { - return; - } - - const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); - if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { - cliError(' gitnexus analyze --watch requires a Git repository.'); - process.exitCode = 1; - return; - } - const repoPath = await fs.realpath(requestedRepoPath); - try { - await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS); - } catch (error) { - cliError(` ${error instanceof Error ? error.message : String(error)}`); - process.exitCode = 1; - return; - } - const baselineEnvironment: WatchEnvironmentBaseline = { - maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, - workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, - verbose: process.env.GITNEXUS_VERBOSE, + let cancelled = false; + let stopWatching: (() => void) | undefined; + const cancel = () => { + cancelled = true; + stopWatching?.(); }; + const onControlMessage = (message: unknown) => { + if ((message as { type?: unknown } | null)?.type === 'cancel') cancel(); + }; + process.on('message', onControlMessage); + process.once('disconnect', cancel); try { - let ignoredConfigSignature: string | undefined; - const reportIgnoredConfig = (names: readonly string[]) => { - const signature = [...names].sort().join(','); - if (signature === ignoredConfigSignature) return; - ignoredConfigSignature = signature; - if (names.length > 0) { - cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); - } - }; - let debounceMs: number; - let analyzeOptions: CoreAnalyzeOptions; + if (process.send) { + if (!process.connected) cancel(); + else + process.send({ type: 'analysis-watch-ready' }, (error) => { + if (error) cancel(); + }); + } + const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { + cliError(' gitnexus analyze --watch requires a Git repository.'); + process.exitCode = 1; + return; + } + const repoPath = await fs.realpath(requestedRepoPath); try { - debounceMs = - positiveInteger( - cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), - '--debounce', - MAX_TIMER_DELAY_MS, - ) ?? DEFAULT_DEBOUNCE_MS; - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); + const admission = resolveHostAnalysisResources({ + totalMb: positiveInteger(cliOptions.resourceBudget, '--resource-budget'), + mainHeapMb: positiveInteger(cliOptions.memoryBudget, '--memory-budget'), + workers: positiveInteger(cliOptions.workers, '--workers'), + includeEmbeddings: await analysisMayEmbed(repoPath, { branch: cliOptions.branch }), + }); + if (cancelled) return; + if ( + await ensureHeap({ + cleanForwardedTermination: true, + memoryBudget: cliOptions.memoryBudget, + admittedHeapMb: admission.mainHeapMb, + }) + ) + return; + await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS); } catch (error) { cliError(` ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; return; } - - let stopWatching!: () => void; - const stopped = new Promise((resolve) => { - stopWatching = resolve; - }); - const stop = () => stopWatching(); - process.once('SIGINT', stop); - process.once('SIGTERM', stop); + const baselineEnvironment: WatchEnvironmentBaseline = { + maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, + workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, + verbose: process.env.GITNEXUS_VERBOSE, + }; try { - let loop: WatchFileLoop; - let fatalRefreshError: unknown; - let configControlValid = true; - let lastSuccessfulRefreshAt: string | undefined; + let ignoredConfigSignature: string | undefined; + const reportIgnoredConfig = (names: readonly string[]) => { + const signature = [...names].sort().join(','); + if (signature === ignoredConfigSignature) return; + ignoredConfigSignature = signature; + if (names.length > 0) { + cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); + } + }; + let debounceMs: number; + let analyzeOptions: CoreAnalyzeOptions; try { - loop = await startWatchFileLoop( + debounceMs = + positiveInteger( + cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), + '--debounce', + MAX_TIMER_DELAY_MS, + ) ?? DEFAULT_DEBOUNCE_MS; + analyzeOptions = await resolveWatchOptions( repoPath, - debounceMs, - async (paths) => { - // Initial validation precedes watcher setup. Read again after it is - // armed so a change during startup cannot leave stale options live. - if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) { - const retryingInvalidConfig = !configControlValid; - try { - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); - configControlValid = true; - } catch (error) { - configControlValid = false; - throw new WatchControlReloadError( - retryingInvalidConfig - ? new Error( - 'Configuration remains invalid; fix it before indexing more changes.', - { - cause: error, - }, - ) - : error, - ); - } - } - const startedAt = Date.now(); - const result = await runFullAnalysis( - repoPath, - analyzeOptions, - { - onProgress: () => {}, - onLog: - process.env.GITNEXUS_VERBOSE === '1' - ? (message) => cliInfo(` ${message}`) - : undefined, - }, - runnerIdentityAtBootstrap, - ); - lastSuccessfulRefreshAt = new Date().toISOString(); - if (paths.length === 0) { - cliInfo( - result.alreadyUpToDate - ? `Watching ${repoPath}; index is up to date.` - : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, - ); - } else { - cliInfo( - refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), - ); - } - }, - (error, paths) => { - const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; - if (shouldStopAfterWatchRefreshFailure(error, paths)) { - fatalRefreshError = error; - const guardTimeout = isIndexLockGuardTimeout(error); - cliError( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - (guardTimeout - ? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.' - : 'Watch mode is stopping because the live index may have been updated in place.'), - guardTimeout - ? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath } - : undefined, - ); - stopWatching(); - return; - } - const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; - cliWarn( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - `Retry scheduled; last success ${lastSuccess}.`, - ); - }, - (error) => { - fatalRefreshError = error; - cliError( - `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + - 'Watch mode is stopping.', - ); - stopWatching(); - }, - process.env[GITNEXUS_RC_ENV]?.trim() || undefined, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, ); } catch (error) { - cliError( - ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, - ); + cliError(` ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; return; } - await stopped; - await loop.close(); - if (fatalRefreshError !== undefined) process.exitCode = 1; + if (cancelled) return; + const stopped = new Promise((resolve) => { + stopWatching = resolve; + }); + const stop = cancel; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + let loop: WatchFileLoop; + let fatalRefreshError: unknown; + let configControlValid = true; + let lastSuccessfulRefreshAt: string | undefined; + try { + loop = await startWatchFileLoop( + repoPath, + debounceMs, + async (paths) => { + if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) { + const retryingInvalidConfig = !configControlValid; + try { + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + configControlValid = true; + } catch (error) { + configControlValid = false; + throw new WatchControlReloadError( + retryingInvalidConfig + ? new Error( + 'Configuration remains invalid; fix it before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + const startedAt = Date.now(); + const result = await runFullAnalysis( + repoPath, + analyzeOptions, + { + onProgress: () => {}, + onLog: + process.env.GITNEXUS_VERBOSE === '1' + ? (message) => cliInfo(` ${message}`) + : undefined, + }, + runnerIdentityAtBootstrap, + ); + lastSuccessfulRefreshAt = new Date().toISOString(); + if (paths.length === 0) { + cliInfo( + result.alreadyUpToDate + ? `Watching ${repoPath}; index is up to date.` + : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, + ); + } else { + cliInfo( + refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), + ); + } + }, + (error, paths) => { + const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; + if (shouldStopAfterWatchRefreshFailure(error, paths)) { + fatalRefreshError = error; + const guardTimeout = isIndexLockGuardTimeout(error); + cliError( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + (guardTimeout + ? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.' + : 'Watch mode is stopping because the live index may have been updated in place.'), + guardTimeout + ? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath } + : undefined, + ); + stopWatching(); + return; + } + const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; + cliWarn( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + `Retry scheduled; last success ${lastSuccess}.`, + ); + }, + (error) => { + fatalRefreshError = error; + cliError( + `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping.', + ); + stopWatching(); + }, + process.env[GITNEXUS_RC_ENV]?.trim() || undefined, + ); + } catch (error) { + cliError( + ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + + await stopped; + await loop.close(); + if (fatalRefreshError !== undefined) process.exitCode = 1; + } finally { + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + } } finally { - process.removeListener('SIGINT', stop); - process.removeListener('SIGTERM', stop); + setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); + setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); } } finally { - setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); - setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); - setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); + process.removeListener('message', onControlMessage); + process.removeListener('disconnect', cancel); } } diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 24d3b2ad4..4e3fbd6a1 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -15,7 +15,6 @@ import v8 from 'v8'; import cliProgress from 'cli-progress'; import { formatAnalyzeFtsSkipSummary } from '../core/search/fts-policy.js'; import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js'; -import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js'; import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js'; import { causeChain } from '../lib/utils.js'; import { @@ -51,8 +50,19 @@ import { GitNexusRcError, } from './analyze-config.js'; import type { AnalyzeOptions } from './analyze-options.js'; -import type { CliPreparedAnalysisMessage } from './analyze-phase-ipc.js'; -import { runFullAnalysis } from '../core/run-analyze.js'; +import { + parseCliPreparedAnalysis, + receiveCliAnalysisContinuation, + type CliAnalysisContinuation, + type CliPreparedAnalysisMessage, +} from './analyze-phase-ipc.js'; +import { AnalysisSupervisorRequired } from '../core/analysis-supervision.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; +import type { AnalysisResourceBudget } from '../core/analysis-resource-budget.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; import { @@ -370,6 +380,8 @@ const shouldBridgeRespawnProgressTty = (): boolean => process.stderr.isTTY === true || process.stdout.isTTY === true; interface RespawnExit { + prepared?: CliPreparedAnalysisMessage; + writerPid?: number; status?: number | null; signal?: NodeJS.Signals | null; stdout?: string; @@ -403,20 +415,31 @@ const appendOutputTail = (tail: string, chunk: unknown): string => { const runRespawnedAnalyze = ( args: readonly string[], env: NodeJS.ProcessEnv, + continuation?: CliAnalysisContinuation, ): Promise => new Promise((resolve) => { let stdout = ''; let stderr = ''; let settled = false; let forwardedSignal: NodeJS.Signals | undefined; + let prepared: CliPreparedAnalysisMessage | undefined; + let phaseError: string | undefined; const child = spawn(process.execPath, [...args], { - stdio: ['inherit', 'pipe', 'pipe'], + stdio: ['inherit', 'pipe', 'pipe', 'ipc'], windowsHide: true, env, }); const forwardSignal = (signal: NodeJS.Signals): void => { forwardedSignal ??= signal; - if (child.exitCode === null && child.signalCode === null) child.kill(signal); + if (child.connected) { + try { + child.send({ type: 'cancel' }, () => { + /* Actual exit remains the release boundary. */ + }); + } catch { + /* A disconnected child cancels at its next safe point. */ + } + } }; const forwardSigint = () => forwardSignal('SIGINT'); const forwardSigterm = () => forwardSignal('SIGTERM'); @@ -425,12 +448,42 @@ const runRespawnedAnalyze = ( settled = true; process.removeListener('SIGINT', forwardSigint); process.removeListener('SIGTERM', forwardSigterm); - resolve({ ...exit, forwardedSignal }); + resolve({ + ...exit, + ...(phaseError ? { status: 1, message: phaseError } : {}), + forwardedSignal, + prepared, + writerPid: child.pid, + }); }; process.once('SIGINT', forwardSigint); process.once('SIGTERM', forwardSigterm); + child.on('message', (message: unknown) => { + try { + const type = (message as { type?: string })?.type; + if (type === 'graph-prepared') { + if (continuation || prepared) throw new Error('Unexpected repeated graph phase'); + prepared = parseCliPreparedAnalysis(message); + } else if (type === 'analysis-continuation-ready' && continuation) { + child.send(continuation, (error) => { + if (error) phaseError = error.message; + }); + } else if ( + (type === 'analysis-ready' || type === 'analysis-watch-ready') && + forwardedSignal + ) { + // Cancellation may arrive while the child is still loading modules. + forwardSignal(forwardedSignal); + } + } catch (error) { + // Never kill a native-active child for malformed IPC. Its exit remains + // the release boundary and a failed projection cannot start a model. + phaseError = error instanceof Error ? error.message : String(error); + } + }); + child.stdout?.on('data', (chunk) => { stdout = appendOutputTail(stdout, chunk); realStdoutWrite(chunk); @@ -682,9 +735,42 @@ export function resolveBudgetHeap(input: BudgetHeapInput): BudgetHeapDecision { * respawned) so the parse phase's remedy text advises by source. Explicit * heap pins and `GITNEXUS_MEMORY=off` intentionally leave it unset. */ export async function ensureHeap( - options: { cleanForwardedTermination?: boolean; memoryBudget?: string } = {}, + options: { + cleanForwardedTermination?: boolean; + memoryBudget?: string; + admittedHeapMb?: number; + } = {}, ): Promise { const nodeOpts = process.env.NODE_OPTIONS || ''; + if ( + options.memoryBudget === undefined && + options.admittedHeapMb !== undefined && + !(Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0) + ) { + const decision = resolveBudgetHeap({ + budgetMb: options.admittedHeapMb, + execArgv: process.execArgv, + nodeOptions: nodeOpts, + autoCapMb: RESPAWN_HEAP_MB, + autopilotDisabled: false, + inheritedSource: 'budget', + }); + if (!decision.respawn) return false; + const ambient = parseMaxOldSpaceMb(nodeOpts); + if (ambient !== null && process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1') { + cliWarn( + ` NODE_OPTIONS pins the heap to ${ambient}MB. Re-running analyze with the admitted ${options.admittedHeapMb}MB main heap.\n`, + ); + } + return respawnWithHeap( + `--max-old-space-size=${decision.sizing.oldSpaceMb}`, + `--max-semi-space-size=${decision.sizing.semiSpaceMb}`, + 'auto', + 'Analysis likely ran out of memory in its admitted main heap. Raise --resource-budget or reduce the analysis scope. If this persists, it may be a native crash unrelated to heap size.', + nodeOpts, + options, + ); + } // The budget is checked BEFORE the GITNEXUS_MEMORY=off return: an explicit // flag is a deliberate per-run choice, which the opt-out does not cover. if (options.memoryBudget !== undefined) { @@ -767,16 +853,16 @@ export async function ensureHeap( /** Run the analyze child with the given heap flags and map its exit onto this process. */ async function respawnWithHeap( - heapFlag: string, - semiFlag: string, - source: HeapLimitSource, + heapFlag: string | undefined, + semiFlag: string | undefined, + source: HeapLimitSource | undefined, oomGuidance: string, nodeOpts: string, options: { cleanForwardedTermination?: boolean }, ): Promise { // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it // only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS. - const cliFlags = [heapFlag, semiFlag]; + const cliFlags = [heapFlag, semiFlag].filter((flag): flag is string => flag !== undefined); if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG); // Preserve the parent's node flags (execArgv) — dropping them breaks any @@ -790,11 +876,28 @@ async function respawnWithHeap( const childArgs = [...preservedExecArgv, ...cliFlags, ...process.argv.slice(1)]; const childEnv = { ...process.env, - NODE_OPTIONS: `${nodeOpts} ${heapFlag} ${semiFlag}`.trim(), + NODE_OPTIONS: [nodeOpts, heapFlag, semiFlag].filter(Boolean).join(' ').trim(), [HEAP_LIMIT_SOURCE_ENV]: source, + GITNEXUS_ANALYSIS_SUPERVISED: '1', + GITNEXUS_ANALYSIS_MANUAL_HEAP: heapFlag === undefined ? '1' : '0', + GITNEXUS_ANALYSIS_SUPERVISOR_BYTES: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES ?? String(analysisSupervisorReservation()), }; if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1'; - const childExit = await runRespawnedAnalyze(childArgs, childEnv); + let childExit = await runRespawnedAnalyze(childArgs, childEnv); + if ( + childExit.status === 0 && + !childExit.signal && + !childExit.forwardedSignal && + childExit.prepared + ) { + if (!childExit.writerPid) throw new Error('Graph process did not report its PID'); + childExit = await runRespawnedAnalyze( + childArgs, + { ...childEnv, GITNEXUS_ANALYSIS_CONTINUATION: '1' }, + { type: 'continue-analysis', writerPid: childExit.writerPid, prepared: childExit.prepared }, + ); + } if (childExit.forwardedSignal !== undefined) { process.exitCode = forwardedSignalExitCode( childExit.forwardedSignal, @@ -831,6 +934,8 @@ async function respawnWithHeap( * for it in the first place. */ const ANALYZE_CLI_ENV_KEYS = [ + 'GITNEXUS_ANALYSIS_MAIN_HEAP_MB', + 'GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', HEAP_LIMIT_SOURCE_ENV, 'GITNEXUS_VERBOSE', 'GITNEXUS_PROFILE_DEFERRED', @@ -904,19 +1009,12 @@ export const analyzeCommand = async ( // Snapshot before ensureHeap: it records GITNEXUS_HEAP_LIMIT_SOURCE on a // kept process, which must not leak into a later programmatic call. const envSnap = snapshotAnalyzeEnv(); - if (await ensureHeap({ memoryBudget: options?.memoryBudget })) return; - forceHeapOOMForTestIfEnabled(); - // Install fatal handlers immediately after re-exec resolution so any + // Install fatal handlers before admission so any // async error that escapes the try/catch below (#1169) surfaces with // a stack trace and a non-zero exit code instead of a silent exit 0. installFatalHandlers(); - // npm-11 npx-crash nudge (#1939). Runs here, after the heap re-exec guard, - // so it fires once in the working process and never on the lazy-startup path - // of other commands (e.g. `gitnexus mcp`). - warnIfNpm11NpxRisk(); - // Snapshot the GITNEXUS_* env vars that the impl writes for downstream // consumption, so they don't leak across `analyzeCommand` invocations in // programmatic callers (tests, long-running hosts). `process.exit(0)` on @@ -1286,6 +1384,67 @@ const analyzeCommandImpl = async ( process.env.GITNEXUS_EMBEDDING_DIMS = dims; } + let resources: AnalysisResourceBudget; + let manualHeap = false; + try { + const supervisorBytes = + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES); + manualHeap = + options.memoryBudget === undefined && + (process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP === '1' || + (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && + (memoryAutopilotDisabled() || + parseMaxOldSpaceMb(process.execArgv.join(' ')) !== null || + (parseMaxOldSpaceMb(process.env.NODE_OPTIONS ?? '') ?? 0) >= RESPAWN_HEAP_MB))); + resources = resolveHostAnalysisResources({ + totalMb: + options.resourceBudget === undefined + ? undefined + : parseIntegerOption(options.resourceBudget, '--resource-budget', { minimum: 1 }), + mainHeapMb: + options.memoryBudget !== undefined + ? parseMemoryBudgetMb(options.memoryBudget) + : process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1' && + process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB !== undefined + ? Number(process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB) + : manualHeap + ? Math.ceil(v8.getHeapStatistics().heap_size_limit / 1024 ** 2) + : undefined, + workers: workerPoolSize, + includeEmbeddings: await analysisMayEmbed(repoPath, { + embeddings: embeddingsEnabled, + force: options.force, + dropEmbeddings: options.dropEmbeddings, + branch: cliOptions?.branch, + }), + supervisorBytes, + }); + // The two child phases retain the same frontend and heap reservation. + // Re-measuring either in the child would add another supervisor or trigger + // a nested heap respawn with a slightly different allocation. + process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB = String(resources.mainHeapMb); + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES = String(supervisorBytes); + if ( + !manualHeap && + (await ensureHeap({ + memoryBudget: options.memoryBudget, + admittedHeapMb: resources.mainHeapMb, + })) + ) + return; + forceHeapOOMForTestIfEnabled(); + } catch (error) { + cliError(` ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + return; + } + + // The npm compatibility warning probes subprocesses. Run it only after the + // heap re-exec guard. A continuation's graph process already emitted it. + if (process.env.GITNEXUS_ANALYSIS_CONTINUATION !== '1') warnIfNpm11NpxRisk(); + // Custom-endpoint UX, emitting at most ONE message that reflects THIS run's // intent (not ambient env). Order matters — the first matching branch wins: // 1. flags given but --embeddings absent: the endpoint won't be used, so @@ -1464,29 +1623,38 @@ const analyzeCommandImpl = async ( bar.start(100, 0, { phase: 'Initializing...' }); - // Graceful SIGINT handling. Pino's default destination is `sync: false` - // (buffered) — flush before exit so in-flight records reach stderr. - // See `gitnexus/src/core/logger.ts:flushLoggerSync`. - let aborted = false; - const sigintHandler = () => { - if (aborted) process.exit(1); - aborted = true; - bar.stop(); - console.log('\n Interrupted — cleaning up...'); - // Bounded CHECKPOINT-then-exit (#2264 review P3): skip the native close (the - // LadybugDB destructor can double-free after --pdg writes), but don't hang - // behind a long --pdg COPY holding the connection lock — bound it so a single - // Ctrl-C stays responsive; the WAL replays on the next analyze. A second - // Ctrl-C (`if (aborted) process.exit(1)` above) remains the escape hatch. - void boundedCheckpointBeforeExit({ - exitCode: 130, - beforeExit: async () => { - const { flushLoggerSync } = await import('../core/logger.js'); - flushLoggerSync(); - }, - }); + // Signals and IPC only request cancellation. The analysis promise owns its + // native calls and cleanup; even a repeated interrupt must not exit mid-call. + const cancellation = new AbortController(); + let cancellationExitCode = 130; + const requestCancellation = (code: number) => { + if (cancellation.signal.aborted) return; + cancellationExitCode = code; + cancellation.abort(new Error('Analysis cancelled')); + console.log('\n Interrupted — waiting for native work to finish...'); }; + const sigintHandler = () => requestCancellation(130); + const sigtermHandler = () => requestCancellation(143); + const cancelMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'cancel') requestCancellation(130); + }; + const disconnected = () => requestCancellation(1); process.on('SIGINT', sigintHandler); + process.on('SIGTERM', sigtermHandler); + process.on('message', cancelMessage); + process.on('disconnect', disconnected); + if (process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1') { + if (!process.connected || !process.send) disconnected(); + else { + try { + process.send({ type: 'analysis-ready' }, (error) => { + if (error) disconnected(); + }); + } catch { + disconnected(); + } + } + } // Route console output through bar.log() to prevent progress bar corruption. // This is a deliberate UI pattern (not a logging concern): analyze runs a @@ -1533,14 +1701,23 @@ const analyzeCommandImpl = async ( } }, 1000); - const t0 = Date.now(); - // ── Run shared analysis orchestrator ─────────────────────────────── try { + // The heap supervisor never runs the pipeline. Load it only in the + // admitted working process, after validation and the re-exec guard. + const analysis = await import('../core/run-analyze.js'); + const continuation = + process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' + ? await receiveCliAnalysisContinuation(cancellation.signal) + : undefined; + cancellation.signal.throwIfAborted(); + const t0 = continuation?.prepared.startedAt ?? Date.now(); const skipAll = options.indexOnly; const skipAgentsMd = skipAll || options.skipAgentsMd; const skipSkills = skipAll || options.skipSkills; const runOptions = { + graphPublication: 'auto' as const, + resources, // The user's own --force only. --skills (skill generation needs a fresh // pipelineResult) and --no-parse-cache (bypassing parser output means // anything only when the pipeline runs) each force the rebuild inside @@ -1605,6 +1782,11 @@ const analyzeCommandImpl = async ( skipNativeCloseOnExit: true, }; const runCallbacks = { + signal: cancellation.signal, + onGraphWork: () => { + if (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' || !process.connected) + throw new AnalysisSupervisorRequired(); + }, onProgress: (_phase, percent, message) => { updateBar(percent, message); }, @@ -1618,11 +1800,36 @@ const analyzeCommandImpl = async ( // further down) writes to them, so selfCommitContextFiles can tell a // pre-existing unstaged user edit apart from this run's stats refresh // and refuse to sweep the former into the latter's commit. - const selfCommitSafety = - options.selfCommit === true + const selfCommitSafety = continuation?.prepared.selfCommitSafety + ? new Map([ + ['AGENTS.md', continuation.prepared.selfCommitSafety.agentsMd], + ['CLAUDE.md', continuation.prepared.selfCommitSafety.claudeMd], + ]) + : options.selfCommit === true ? snapshotSelfCommitSafety(repoPath, ['AGENTS.md', 'CLAUDE.md']) : undefined; - const result = await runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); + const result = continuation + ? { + ...continuation.prepared.result, + stats: + ( + await analysis.continuePreparedAnalysis( + continuation.prepared.handoff, + continuation.writerPid, + runOptions, + runCallbacks, + cancellation.signal, + ) + ).stats ?? {}, + rebuildReasons: [], + pipelineResult: undefined, + preparedGraph: undefined, + alreadyUpToDate: false, + ftsRepairedOnly: false, + graphWriteCollapsed: undefined, + } + : await analysis.runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); + cancellation.signal.throwIfAborted(); let preparedMessage: CliPreparedAnalysisMessage | undefined; if (result.preparedGraph) { @@ -1870,6 +2077,31 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); + if (cancellation.signal.aborted) { + process.exitCode = cancellationExitCode; + return; + } + + if (err instanceof AnalysisSupervisorRequired) { + const sizing = resolveBudgetHeap({ + budgetMb: resources.mainHeapMb, + execArgv: process.execArgv, + nodeOptions: process.env.NODE_OPTIONS ?? '', + autoCapMb: RESPAWN_HEAP_MB, + autopilotDisabled: false, + inheritedSource: 'budget', + }).sizing; + await respawnWithHeap( + manualHeap ? undefined : `--max-old-space-size=${sizing.oldSpaceMb}`, + manualHeap ? undefined : `--max-semi-space-size=${sizing.semiSpaceMb}`, + manualHeap ? (process.env[HEAP_LIMIT_SOURCE_ENV] as HeapLimitSource | undefined) : 'auto', + 'Analysis exceeded its resource budget.', + process.env.NODE_OPTIONS ?? '', + {}, + ); + return; + } + const msg = err instanceof Error ? err.message : String(err); // Registry name-collision from --name (#829) — surface as an @@ -2223,6 +2455,12 @@ const analyzeCommandImpl = async ( process.exitCode = 1; return; + } finally { + clearInterval(elapsedTimer); + process.removeListener('SIGINT', sigintHandler); + process.removeListener('SIGTERM', sigtermHandler); + process.removeListener('message', cancelMessage); + process.removeListener('disconnect', disconnected); } // LadybugDB's native module holds open handles that prevent Node from exiting. diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts index b4b6626fb..02369b8da 100644 --- a/gitnexus/src/cli/embeddings-sync.ts +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -1,36 +1,17 @@ import { lstat } from 'node:fs/promises'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; +import { resolveGraphPath } from '../storage/shared-store.js'; +import { resolveHostAnalysisResources } from '../core/analysis-host-resources.js'; +import { readPreparedAnalysis } from '../core/analysis-publication.js'; import path from 'node:path'; +import { getHeapStatistics } from 'node:v8'; import { cliInfo } from './cli-message.js'; import { getGitRoot } from '../storage/git.js'; -import { - acquireIndexLock, - requireExclusiveIndexLock, - sweepStagingArtifacts, -} from '../storage/index-lock.js'; -import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'; -import { - closeLbug, - executeQuery, - executeWithReusedStatement, - fetchExistingEmbeddingHashes, - initLbug, -} from '../core/lbug/lbug-adapter.js'; -import { - decideEmbeddingResume, - mintInterruptedCheckpoint, - mintPartialCheckpoint, - mintUnverifiedCountCheckpoint, - type EmbeddingCheckpoint, - type EmbeddingCheckpointProgress, -} from '../core/embedding-checkpoint.js'; +import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; +import { getStoragePaths, loadMeta } from '../storage/repo-manager.js'; +import { decideEmbeddingResume } from '../core/embedding-checkpoint.js'; import { EMBEDDING_DIMS, embeddingDimsMismatch } from '../core/lbug/schema.js'; -import type { RepoMeta } from '../storage/repo-meta.js'; -import { - measurePersistedEmbeddingCount, - persistedEmbeddingCountOrUndefined, -} from '../core/embedding-count.js'; import { isHttpMode } from '../core/embeddings/http-client.js'; import { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS, @@ -42,8 +23,6 @@ import { assessLocalEmbeddingRuntime, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js'; -import { reapEmbeddingSidecarSafely } from '../core/embeddings/embedding-sidecar-reap.js'; -import { contentRetentionFromMeta } from '../core/content-retention.js'; /** Add missing embeddings directly to a healthy index, checkpointing periodically. */ export const embeddingsSyncCommand = async (inputPath?: string): Promise => { @@ -54,8 +33,12 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = const metaDir = path.dirname(metaPath); // Writes go to the slot's own graph. A shared-store checkout that reads an // immutable commit graph (#3352) takes a private copy first. - const lbugPath = path.join(metaDir, LBUG_DIRECTORY); - const lock = await acquireIndexLock(metaDir, { sweep: false }); + const resources = resolveHostAnalysisResources({ + includeParsing: false, + includeEmbeddings: true, + mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + }); + const lock = await acquireAnalysisSlotLock(metaDir, { sweep: false }); try { requireExclusiveIndexLock( lock, @@ -70,6 +53,8 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = 'Run `gitnexus analyze` to recover them first.', ); } + if (await readPreparedAnalysis(metaDir)) + throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.'); sweepStagingArtifacts(metaDir); if (!(await ensurePrivateSharedGraph(metaDir, (m) => console.log(` ${m}`)))) { throw new Error('The shared graph this checkout reads is gone. Run gitnexus analyze first.'); @@ -81,6 +66,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = throw new Error('The structural index is incomplete. Run gitnexus analyze --force first.'); } + const lbugPath = resolveGraphPath(metaDir); let lbugStat; try { lbugStat = await lstat(lbugPath); @@ -97,8 +83,6 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js'); const identity = resolveEmbeddingIdentity(); - let forceReembedNodeIds: ReadonlySet | undefined; - let resumedFrom: EmbeddingCheckpoint | undefined; if (meta.embeddingCheckpoint) { const checkpoint = meta.embeddingCheckpoint; const decision = decideEmbeddingResume(checkpoint, identity); @@ -125,25 +109,15 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = ); } cliInfo(decision.log); - if (decision.action === 'resume') { - forceReembedNodeIds = decision.pendingNodeIds; - resumedFrom = decision.resumedFrom; - } } - // The vector column is FLOAT[N] fixed when the index was built, and the - // pipeline deletes each batch's stale rows immediately before inserting the - // replacements — so a width change here deletes rows it cannot re-insert. - // `analyze` forces a full rebuild on the same mismatch; only a rebuild can - // retype the column, so this writer refuses instead. An absent recorded - // width is not a mismatch (see `embeddingDimsMismatch`). - if (embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) { + // Keep legacy sync's explicit migration requirement; the new vector store + // can use a different width once analyze has established its own generation. + if (!meta.graphGeneration && embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) throw new Error( `Cannot sync embeddings: this index stores FLOAT[${meta.embeddingDims}] vectors, ` + - `but this run embeds at ${EMBEDDING_DIMS} dimensions. ` + - 'Run `gitnexus analyze --embeddings --force` to rebuild the column at the new width.', + `but this run embeds at ${EMBEDDING_DIMS} dimensions. Run gitnexus analyze --embeddings --force.`, ); - } if (!isHttpMode()) { const assessment = assessLocalEmbeddingRuntime(); @@ -167,93 +141,27 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = } } - await initLbug(lbugPath); - try { - const existing = await fetchExistingEmbeddingHashes(executeQuery); - let lastPercent = -1; - - const countEmbeddings = async (): Promise => - persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery)); - // One write path for every meta update this command makes. The re-read - // happens immediately before each save so a concurrent writer's fields - // survive. #2790 traced two production drifts to hand-copied writers of - // these exact fields, so this file keeps one copy instead of three. - const persistMeta = async (patch: (latest: RepoMeta) => Partial): Promise => { - const latest = (await loadMeta(metaDir)) ?? meta; - await saveMeta(metaDir, { ...latest, ...patch(latest) }); - }; - const saveCheckpoint = async ( - checkpoint: EmbeddingCheckpointProgress, - pendingNodeIds: string[], - embeddings?: number, - ): Promise => - persistMeta((latest) => ({ - ...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }), - embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds), - })); - - cliInfo(`Embedding ${repoPath}`); - cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`); - - const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js'); - const result = await runEmbeddingPipeline( - executeQuery, - executeWithReusedStatement, - (progress) => { - const percent = Math.floor(progress.percent); - if (percent !== lastPercent && (percent % 5 === 0 || percent === 100)) { - lastPercent = percent; - cliInfo( - ` ${percent}% — ${progress.nodesProcessed ?? 0}/${progress.totalNodes ?? '?'} nodes`, - ); + cliInfo(`Embedding ${repoPath}`); + let lastPercent = -1; + const { runAnalysisEmbeddingPhase } = await import('../core/analysis-embedding-phase.js'); + const completed = await runAnalysisEmbeddingPhase({ + metadataDirectory: metaDir, + graphPath: lbugPath, + resources, + nodeLimit: 0, + expectedGeneration: meta.graphGeneration?.id, + callbacks: { + onLog: cliInfo, + onProgress: (_phase, percent, message) => { + const rounded = Math.floor(percent); + if (rounded !== lastPercent) { + lastPercent = rounded; + cliInfo(` ${rounded}% — ${message}`); } }, - {}, - undefined, - existing && existing.size ? existing : undefined, - { - source: { - repoPath, - fileHashes: meta.fileHashes, - contentRetention: contentRetentionFromMeta(meta), - }, - forceReembedNodeIds, - onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { - await saveCheckpoint(checkpoint, nodeIds); - }, - onCheckpoint: async (checkpoint) => { - await saveCheckpoint(checkpoint, [], await countEmbeddings()); - }, - }, - ); - - const embeddings = await countEmbeddings(); - if (embeddings === undefined) { - // Keep last-known stats.embeddings. An interrupted window marker would - // fail the identity gate on the next run even though this run finished; - // unverified-count is the recovery kind that forces a recount (#2790). - await persistMeta(() => ({ - embeddingCheckpoint: result.failedNodeIds.length - ? mintPartialCheckpoint(identity, result, resumedFrom) - : mintUnverifiedCountCheckpoint(identity, { - nodesProcessed: result.nodesProcessed, - totalNodes: result.nodesProcessed, - chunksProcessed: result.chunksProcessed, - }), - })); - throw new Error('Could not verify persisted embedding count.'); - } - await persistMeta((latest) => ({ - stats: { ...latest.stats, embeddings }, - embeddingCheckpoint: result.failedNodeIds.length - ? mintPartialCheckpoint(identity, result, resumedFrom) - : undefined, - })); - cliInfo(`Embeddings ready: ${embeddings}`); - } finally { - await closeLbug().catch(() => {}); - await reapEmbeddingSidecarSafely(); - } + }, + }); + cliInfo(`Embeddings ready: ${completed.stats?.embeddings ?? 0}`); } finally { lock.release(); } diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index d89641332..25a398347 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -178,6 +178,10 @@ program 'exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size ' + 'pin; parse workers keep their own heap caps.', ) + .option( + '--resource-budget ', + 'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.', + ) .option( '--workers ', 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', diff --git a/gitnexus/src/core/analysis-embedding-phase.ts b/gitnexus/src/core/analysis-embedding-phase.ts new file mode 100644 index 000000000..4e4e2c99a --- /dev/null +++ b/gitnexus/src/core/analysis-embedding-phase.ts @@ -0,0 +1,264 @@ +/** Common bounded embedding phase. Callers retain the ordinary slot writer lock. */ +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import lbug from '@ladybugdb/core'; +import type { AnalyzeCallbacks } from './run-analyze.js'; +import type { AnalysisResourceBudget } from './analysis-resource-budget.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { loadMeta, saveMeta } from '../storage/repo-manager.js'; +import { SCHEMA_FINGERPRINT } from './lbug/schema.js'; +import { openLbugConnection, closeLbugConnection } from './lbug/lbug-config.js'; +import { drainAnalysisNativeResources } from './lbug/lbug-adapter.js'; +import { assertLbugPublicationReady } from './lbug/publication.js'; +import { exportLegacyVectors } from './lbug/legacy-vector-export.js'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from './analysis-publication.js'; +import { verifyPublishedAnalysisGraph } from './analysis-graph-verification.js'; +import { runPublishedEmbeddingJob } from './embeddings/embedding-job.js'; +import { drainEmbeddingSidecarForAnalysis } from './embeddings/embedding-sidecar-client.js'; +import { deriveEmbeddingCap } from './embedding-mode.js'; +import { embeddingProgressFromState, type EmbeddingJobProgress } from './analysis-progress.js'; + +export function canPromoteLegacyGraph(metadata: RepoMeta): boolean { + return ( + !metadata.graphGeneration && + !metadata.incrementalInProgress && + !metadata.graphWriteCollapsed && + !metadata.embeddingCheckpoint?.recovery && + metadata.schemaFingerprint === SCHEMA_FINGERPRINT && + metadata.runnerIdentity?.schemaVersion === 4 && + metadata.parseCacheIdentity !== undefined && + metadata.fileHashes !== undefined + ); +} + +/** Adopt a healthy legacy graph without parsing, rewriting or copying its graph. + * Only its vector table is exported, retaining historical provenance for the + * complete-group validator. Interrupted staged graphs take analyze's existing + * recovery path instead. */ +export async function promoteLegacyGraph( + directory: string, + graphPath: string, + metadata: RepoMeta, + resources: AnalysisResourceBudget, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!canPromoteLegacyGraph(metadata)) + throw new Error( + 'Legacy graph provenance requires analysis before embedding; run gitnexus analyze first.', + ); + if ( + path.dirname(graphPath) !== path.resolve(directory) || + path.basename(graphPath) !== (metadata.graphPath ?? 'lbug') + ) + throw new Error('Legacy graph promotion requires an owned private graph'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + signal?.throwIfAborted(); + await assertLbugPublicationReady(graphPath); + const handle = await openLbugConnection(lbug, graphPath, { + readOnly: true, + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }); + let legacy = metadata.legacyVectorSource; + try { + if (!legacy) { + const historical = metadata.embeddingCheckpoint; + legacy = await exportLegacyVectors({ + metadataDirectory: directory, + query: async (sql) => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected legacy migration result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }, + provenance: { + schemaFingerprint: metadata.schemaFingerprint!, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: historical?.pendingNodeIds ?? [], + bufferPoolBytes: resources.embedding.vectorPoolBytes, + threads: resources.embedding.nativeThreads, + signal, + }); + } + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graphPath }); + } + signal?.throwIfAborted(); + // Persist the independent paid source before retiring the monolithic name. + const retained = { ...metadata, ...(legacy ? { legacyVectorSource: legacy } : {}) }; + // Canonical legacy paths are represented by absence in the publication + // contract; shared-store privatization can explicitly record that same name. + if (retained.graphPath === 'lbug') delete retained.graphPath; + await saveMeta(directory, retained); + const now = Date.now(); + const dirty = await beginInPlaceAnalysis( + directory, + { + ...retained, + incrementalInProgress: { + startedAt: now, + updatedAt: now, + phase: 'legacy-migration', + toWriteCount: 0, + }, + }, + (meta) => saveMeta(directory, meta), + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Legacy migration has no owned publication attempt'); + const content = createHash('sha256'); + for (const entry of Object.entries(metadata.fileHashes!).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + )) + content.update(JSON.stringify(entry)).update('\0'); + await prepareAnalysisPublication(directory, graphPath, retained, content.digest('hex'), { + mode: 'in-place', + attemptId, + }); + const published = await publishPreparedAnalysis(directory, { + saveMetadata: (meta) => saveMeta(directory, meta), + verify: (file, meta) => + verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }), + }); + if (!published) throw new Error('Legacy graph publication disappeared'); + return published; +} + +export interface AnalysisEmbeddingPhaseOptions { + metadataDirectory: string; + graphPath: string; + resources: AnalysisResourceBudget; + callbacks: AnalyzeCallbacks; + nodeLimit?: number; + signal?: AbortSignal; + /** Reject a superseded handoff before any model or paid provider request. */ + expectedGeneration?: string; +} + +export async function runAnalysisEmbeddingPhase( + options: AnalysisEmbeddingPhaseOptions, +): Promise { + const { metadataDirectory, resources, callbacks, signal } = options; + signal?.throwIfAborted(); + let metadata = await loadMeta(metadataDirectory); + if (await readPreparedAnalysis(metadataDirectory)) + throw new Error('A prepared graph awaits publication; run gitnexus analyze before embedding.'); + if (!metadata || metadata.incrementalInProgress || metadata.graphWriteCollapsed) + throw new Error('The graph is incomplete; run gitnexus analyze before embedding.'); + if (!Number.isSafeInteger(metadata.stats?.nodes) || (metadata.stats?.nodes ?? -1) < 0) + throw new Error('The graph node count is unverified; run gitnexus analyze before embedding.'); + if (options.expectedGeneration && metadata.graphGeneration?.id !== options.expectedGeneration) + throw new Error('Graph generation changed before embedding; retry analysis.'); + const cap = deriveEmbeddingCap( + metadata.stats?.nodes ?? 0, + metadata.embeddingJob || metadata.embeddingCheckpoint ? 0 : options.nodeLimit, + ); + if (cap.skipForCap) { + callbacks.onLog?.( + `Embeddings skipped: graph exceeds the ${cap.nodeLimit}-node cap; use --embeddings 0 to disable it.`, + ); + await callbacks.onGraphPublished?.(); + return metadata; + } + const { isHttpMode } = await import('./embeddings/http-client.js'); + if (!isHttpMode() && resources.embedding.modelBytes === 0) + throw new Error( + 'Local embedding work was not admitted in this resource budget; retry analysis.', + ); + if (metadata.embeddingCheckpoint) { + const { decideEmbeddingResume } = await import('./embedding-checkpoint.js'); + const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js'); + const identity = resolveEmbeddingIdentity(); + const checkpoint = metadata.embeddingCheckpoint; + if (Object.hasOwn(checkpoint, 'recovery')) + throw new Error('The checkpoint references staged embeddings; run gitnexus analyze first.'); + const decision = decideEmbeddingResume(checkpoint, identity); + if (decision.action === 'abort') throw new Error(decision.error); + if ( + checkpoint.provider !== identity.provider || + checkpoint.model !== identity.model || + checkpoint.dimensions !== identity.dimensions + ) + throw new Error( + 'The embedding checkpoint has a different provider, model or width; run gitnexus analyze --embeddings --force.', + ); + callbacks.onLog?.(decision.log); + } + if (!metadata.graphGeneration) + metadata = await promoteLegacyGraph( + metadataDirectory, + options.graphPath, + metadata, + resources, + signal, + ); + const graphPath = path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'); + let progress: EmbeddingJobProgress = { phase: 'preparing' }; + callbacks.onProgress('graph-ready', 90, 'Graph available; preparing embeddings', { + graphPhase: 'ready', + embeddingProgress: progress, + }); + try { + await callbacks.onGraphPublished?.(); + const result = await runPublishedEmbeddingJob({ + metadataDirectory, + graphPath, + resources: resources.embedding, + signal, + config: { threads: resources.embedding.nativeThreads }, + log: callbacks.onLog, + metadata: { + loadMetadata: () => loadMeta(metadataDirectory), + saveMetadata: (meta) => saveMeta(metadataDirectory, meta), + }, + onProgress: (state) => { + progress = embeddingProgressFromState(state); + const completed = state.completedNodes + state.reusedNodes; + const percent = + state.phase === 'indexing' ? 99 : 90 + 8 * (completed / Math.max(1, state.totalNodes)); + callbacks.onProgress( + 'embeddings', + percent, + state.phase === 'indexing' + ? 'Indexing completed embeddings' + : `Embedding ${completed}/${state.totalNodes}`, + { graphPhase: 'ready', embeddingProgress: progress }, + ); + }, + }); + if (result.phase !== 'ready') + throw new Error( + 'Embedding work remains incomplete; the graph is available. Retry to resume durable progress.', + ); + return result.metadata; + } catch (error) { + callbacks.onProgress('embeddings', 98, 'Embeddings incomplete; graph remains available', { + graphPhase: 'ready', + embeddingProgress: { ...progress, phase: 'failed' }, + }); + throw error; + } +} diff --git a/gitnexus/src/core/analysis-graph-verification.ts b/gitnexus/src/core/analysis-graph-verification.ts new file mode 100644 index 000000000..fdc471d6d --- /dev/null +++ b/gitnexus/src/core/analysis-graph-verification.ts @@ -0,0 +1,70 @@ +/** Read-only verification after the graph writer has physically released its DB. */ +import lbug from '@ladybugdb/core'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_FINGERPRINT } from './lbug/schema.js'; +import { closeLbugConnection, openLbugConnection } from './lbug/lbug-config.js'; +import { assertLbugPublicationReady } from './lbug/publication.js'; +import { getFtsIndexes } from './search/fts-schema.js'; +import { verifySearchFTSIndexes } from './search/fts-indexes.js'; + +function count(value: unknown): number { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Published graph count is unverified'); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) + throw new Error('Published graph count is invalid'); + return result; +} + +export async function verifyPublishedAnalysisGraph( + graphPath: string, + metadata: RepoMeta, + resources: { bufferPoolBytes: number; threads: number }, +): Promise { + if ( + metadata.incrementalInProgress || + metadata.graphWriteCollapsed || + metadata.schemaFingerprint !== SCHEMA_FINGERPRINT || + !metadata.graphGeneration + ) + throw new Error('Prepared graph has no compatible, verified schema generation'); + const expectedNodes = count(metadata.stats?.nodes); + const expectedEdges = count(metadata.stats?.edges); + await assertLbugPublicationReady(graphPath); + const handle = await openLbugConnection(lbug, graphPath, { ...resources, readOnly: true }); + const query = async (sql: string): Promise[]> => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected publication verification result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }; + try { + // Binding every structural table also rejects incomplete native catalogs. + // Count metadata is bounded; no source text, graph rows or vectors enter JS. + const nodes = await query( + NODE_TABLES.map((table) => `MATCH (n:\`${table}\`) RETURN count(n) AS count`).join( + ' UNION ALL ', + ), + ); + const edges = await query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS count`); + if ( + nodes.length !== NODE_TABLES.length || + edges.length !== 1 || + nodes.reduce((total, row) => total + count(row.count), 0) !== expectedNodes || + count(edges[0].count) !== expectedEdges + ) + throw new Error('Prepared graph counts differ after writer release; artifacts retained'); + if (metadata.capabilities?.fts.status === 'available') { + await query('LOAD FTS'); + const missing = await verifySearchFTSIndexes(query, getFtsIndexes(metadata.ftsProfile)); + if (missing.length) + throw new Error(`Prepared graph search indexes are incomplete: ${missing.join(', ')}`); + } + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graphPath }); + } +} diff --git a/gitnexus/src/core/analysis-host-resources.ts b/gitnexus/src/core/analysis-host-resources.ts new file mode 100644 index 000000000..d5f11c088 --- /dev/null +++ b/gitnexus/src/core/analysis-host-resources.ts @@ -0,0 +1,81 @@ +/** Host inputs for the pure phase-admission policy. No native/model imports. */ +import os from 'node:os'; +import path from 'node:path'; +import { effectiveRamBytes } from './ingestion/utils/effective-ram.js'; +import { + resolveAnalysisResourceBudget, + type AnalysisResourceBudget, + type AnalysisResourceRequest, +} from './analysis-resource-budget.js'; + +const MIB = 1024 * 1024; + +function positiveEnvironment(name: string): number | undefined { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === '') return undefined; + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error(`${name} must be a positive safe integer for bounded analysis`); + return value; +} + +export interface HostAnalysisResourceOptions { + totalMb?: number; + mainHeapMb?: number; + workers?: number; + includeParsing?: boolean; + /** Local model capacity is only reserved when this invocation may embed. */ + includeEmbeddings: boolean; + supervisorBytes?: number; + /** Share one host envelope among concurrent auto-sync analyses. */ + concurrency?: number; +} + +export function resolveHostAnalysisResources( + options: HostAnalysisResourceOptions, +): AnalysisResourceBudget { + const concurrency = options.concurrency ?? 1; + if (!Number.isSafeInteger(concurrency) || concurrency < 1) + throw new Error('Analysis concurrency must be a positive safe integer'); + const http = Boolean(process.env.GITNEXUS_EMBEDDING_URL && process.env.GITNEXUS_EMBEDDING_MODEL); + const request: AnalysisResourceRequest = { + effectiveMemoryBytes: Math.floor(effectiveRamBytes() / concurrency), + availableCpus: Math.max(1, Math.floor(os.availableParallelism() / concurrency)), + totalMb: options.totalMb, + mainHeapMb: options.mainHeapMb, + allowHeapOvercommit: options.mainHeapMb !== undefined && options.totalMb === undefined, + workers: options.workers ?? positiveEnvironment('GITNEXUS_WORKER_POOL_SIZE'), + workerHeapMb: positiveEnvironment('GITNEXUS_WORKER_HEAP_MB'), + embeddingThreads: positiveEnvironment('GITNEXUS_EMBEDDING_THREADS'), + preferredWorkers: 16, + includeParsing: options.includeParsing, + graphPoolBytes: positiveEnvironment('GITNEXUS_LBUG_BUFFER_POOL_SIZE'), + modelMemoryMb: options.includeEmbeddings && !http ? 2048 : 0, + supervisorBytes: options.supervisorBytes, + }; + return resolveAnalysisResourceBudget(request); +} + +/** Charge the retained frontend once, including a floor for its IPC/runtime. */ +export function analysisSupervisorReservation(): number { + return Math.max(192 * MIB, Math.ceil(process.memoryUsage().rss / MIB) * MIB); +} + +/** Cheap preflight only. Core still revalidates the exact slot under its lock. */ +export async function analysisMayEmbed( + repoPath: string, + options: { embeddings?: boolean; force?: boolean; dropEmbeddings?: boolean; branch?: string }, +): Promise { + if (options.embeddings) return true; + if (options.dropEmbeddings) return false; + const { getStoragePaths, loadMeta } = await import('../storage/repo-manager.js'); + const { resolveBranchPlacement } = await import('../storage/branch-index.js'); + const placement = await resolveBranchPlacement(repoPath, options.branch ?? null); + const directory = path.dirname(getStoragePaths(repoPath, placement.branch).metaPath); + const meta = await loadMeta(directory); + return Boolean( + meta?.embeddingJob || + meta?.embeddingCheckpoint || + (options.force && (meta?.stats?.embeddings ?? 0) > 0), + ); +} diff --git a/gitnexus/src/core/analysis-parse-cache.ts b/gitnexus/src/core/analysis-parse-cache.ts index a6a8cbcd8..a45c87178 100644 --- a/gitnexus/src/core/analysis-parse-cache.ts +++ b/gitnexus/src/core/analysis-parse-cache.ts @@ -21,6 +21,8 @@ interface CachePublicationOptions { storagePath: string; metadataDirectory: string; cacheRoot: string; + /** The exact published graph's handoff; never scan generation history. */ + publishedGeneration?: string; sharedStore?: SharedStoreLayout; collectBranchKeys: ( root: string, @@ -91,6 +93,11 @@ export async function publishAnalysisParseCache( ); }; await (sharedStore ? withStoreLock(sharedStore, 'cache', publish) : publish()); + if (options.publishedGeneration) { + const root = await embeddingJobDirectory(metadataDirectory); + await fs.rm(path.join(root, receiptName(options.publishedGeneration)), { force: true }); + await syncEmbeddingJobDirectory(root); + } } // This contains only chunk keys, never ASTs, graph data or source content. @@ -240,7 +247,11 @@ export async function loadAnalysisParseCache( ) throw new Error('Prepared parser cache does not match the published graph'); const usedKeys = new Set(metadata.cacheKeys ?? []); - if (payload.used.length !== usedKeys.size || payload.used.some((key) => !usedKeys.has(key))) + if ( + payload.used.length !== usedKeys.size || + new Set(payload.used).size !== usedKeys.size || + payload.used.some((key) => !usedKeys.has(key)) + ) throw new Error('Prepared parser cache selection differs from the published graph'); return { identity: payload.identity, diff --git a/gitnexus/src/core/analysis-phase-supervisor.ts b/gitnexus/src/core/analysis-phase-supervisor.ts index 66dd46b66..4f4d66a4c 100644 --- a/gitnexus/src/core/analysis-phase-supervisor.ts +++ b/gitnexus/src/core/analysis-phase-supervisor.ts @@ -2,7 +2,8 @@ import path from 'node:path'; import type { ChildProcess } from 'node:child_process'; import type { RepoMeta } from '../storage/repo-meta.js'; -import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { embeddingJobDirectory } from '../storage/embedding-job.js'; import { publishPreparedAnalysis, @@ -126,7 +127,7 @@ export async function publishAnalysisAfterProcessExit( actions.signal?.throwIfAborted(); requireSameTarget(expected, await actions.resolveTarget()); const root = await embeddingJobDirectory(expected.metadataDirectory); - const lock = await acquireIndexLock(root, { + const lock = await acquireAnalysisSlotLock(root, { sweep: false, log: actions.log, onWaitStart: actions.onWaitStart, diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 63a2f87e0..65971af62 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -13,6 +13,9 @@ export interface AnalysisResourceRequest { /** Explicit total reservation ceiling; independent of the heap-only flag. */ totalMb?: number; mainHeapMb?: number; + /** Honor an existing heap override when no explicit family ceiling was set. + * This may reserve virtual memory above RAM; native defaults still use RAM. */ + allowHeapOvercommit?: boolean; workers?: number; workerHeapMb?: number; embeddingThreads?: number; @@ -78,7 +81,7 @@ export function resolveAnalysisResourceBudget( : integer(request.embeddingThreads, 'embedding threads'); if (embeddingThreads > cpus) throw new Error('Embedding threads exceed the CPU allowance'); const availableMb = Math.floor(memory / MIB); - const totalMb = + let totalMb = request.totalMb === undefined ? Math.floor(availableMb * 0.75) : integer(request.totalMb, 'resource budget'); @@ -87,7 +90,6 @@ export function resolveAnalysisResourceBudget( `Resource budget ${totalMb} MiB exceeds effective process memory ${availableMb} MiB`, ); } - const totalBytes = bytes(totalMb, 'resource budget'); const headroomMb = Math.max(128, Math.ceil(totalMb * 0.12)); const supervisorMb = Math.ceil( integer(request.supervisorBytes ?? 0, 'supervisor reservation', 0) / MIB, @@ -114,11 +116,24 @@ export function resolveAnalysisResourceBudget( supervisorMb + graphPoolMb + (includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0); - const maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); + let maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); const mainHeapMb = request.mainHeapMb === undefined ? Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) : integer(request.mainHeapMb, 'main heap reservation'); + // --memory-budget predates the family budget and deliberately permits a heap + // above RAM (with the CLI's existing swap warning). Extend only the finite + // heap reservation; do not pretend the host has more RAM or enlarge native + // pools/queues. An explicit --resource-budget always remains authoritative. + if ( + request.allowHeapOvercommit && + request.totalMb === undefined && + request.mainHeapMb !== undefined + ) { + totalMb += Math.max(0, mainHeapMb - maximumMainMb); + maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); + } + const totalBytes = bytes(totalMb, 'resource budget'); // --memory-budget has an established 200 MiB minimum. Preserve that explicit // choice; the larger floor is only the automatic sizing policy. const minimumMainMb = request.mainHeapMb === undefined ? MIN_MAIN_HEAP_MB : 200; diff --git a/gitnexus/src/core/analysis-supervision.ts b/gitnexus/src/core/analysis-supervision.ts new file mode 100644 index 000000000..bfeca331d --- /dev/null +++ b/gitnexus/src/core/analysis-supervision.ts @@ -0,0 +1,7 @@ +/** Raised before native graph work when a CLI needs its phase supervisor. */ +export class AnalysisSupervisorRequired extends Error { + constructor() { + super('Graph work requires an analysis supervisor'); + this.name = 'AnalysisSupervisorRequired'; + } +} diff --git a/gitnexus/src/core/augmentation/engine.ts b/gitnexus/src/core/augmentation/engine.ts index ef6fe0c8c..8393990eb 100644 --- a/gitnexus/src/core/augmentation/engine.ts +++ b/gitnexus/src/core/augmentation/engine.ts @@ -21,7 +21,6 @@ import { requireRegisteredStoragePath, STATUS_STORAGE_REQUIREMENTS, } from '../../storage/storage-resolver.js'; -import { LBUG_DIRECTORY } from '../../storage/storage-constants.js'; import { BRANCHES_DIR, branchSlug } from '../../storage/branch-index.js'; import { getCurrentBranch } from '../../storage/git.js'; import { escapeCypherString } from '../lbug/cypher-escape.js'; @@ -85,10 +84,7 @@ async function findRepoForCwd(cwd: string): Promise<{ return { name: bestMatch.name, storagePath, - lbugPath: - indexDir === storagePath - ? resolveGraphPath(storagePath) - : path.join(indexDir, LBUG_DIRECTORY), + lbugPath: resolveGraphPath(indexDir), }; } catch { return null; diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 077e74891..e2b5a0a91 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -4,8 +4,18 @@ import { createRequire } from 'node:module'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; -import type { WorkerMessage } from '../../server/analyze-worker-protocol.js'; +import type { + WorkerMessage, + ContinueMessage, + GraphPreparedMessage, +} from '../../server/analyze-worker-protocol.js'; import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../analysis-host-resources.js'; +import { projectPreparedAnalysisHandoff } from '../analysis-phase-supervisor.js'; const _require = createRequire(import.meta.url); export type AutoSyncAnalysisRunner = ( @@ -20,6 +30,7 @@ export type AutoSyncAnalysisRunner = ( ) => Promise>; interface AnalysisWorker extends Pick { + pid?: number; stdout?: Pick | null; stderr?: Pick | null; unref?: () => void; @@ -95,8 +106,25 @@ export function createAutoSyncAnalysisRunner( overrides: Partial = {}, ): AutoSyncAnalysisRunner { const deps = { ...DEFAULT_DEPS, ...overrides }; - return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency, onProgress) => - new Promise>((resolve, reject) => { + return async ( + repoPath, + options, + timeoutMs, + signal, + onCancellationRequested, + concurrency, + onProgress, + ) => { + signal?.throwIfAborted(); + const resources = + options.resources ?? + resolveHostAnalysisResources({ + includeEmbeddings: await analysisMayEmbed(repoPath, options), + supervisorBytes: Math.ceil(analysisSupervisorReservation() / (concurrency ?? 1)), + concurrency, + }); + const analysisOptions = { ...options, graphPublication: 'auto' as const, resources }; + return new Promise>((resolve, reject) => { if (signal?.aborted) { reject(new Error('Analysis cancelled.')); return; @@ -112,17 +140,16 @@ export function createAutoSyncAnalysisRunner( reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); return; } - const workerHeapMb = resolveWorkerHeapMb(concurrency); + const workerHeapMb = resources.mainHeapMb - 48; const execArgv = isDev ? [ '--import', pathToFileURL(_require.resolve('tsx/esm')).href, `--max-old-space-size=${workerHeapMb}`, + '--max-semi-space-size=16', ] - : [`--max-old-space-size=${workerHeapMb}`]; - const child = deps.forkWorker(workerPath, execArgv); - child.stdout?.resume(); - child.stderr?.resume(); + : [`--max-old-space-size=${workerHeapMb}`, '--max-semi-space-size=16']; + let child: AnalysisWorker; let terminalOutcome: WorkerMessage | undefined; let terminationError: Error | undefined; @@ -138,8 +165,8 @@ export function createAutoSyncAnalysisRunner( // so both handles have to go. Never a kill: the child may be inside // native work and is left to reach its own safe point. const releaseChild = () => { - child.channel?.unref?.(); - child.unref?.(); + child?.channel?.unref?.(); + child?.unref?.(); }; const settle = ( error?: Error, @@ -159,7 +186,7 @@ export function createAutoSyncAnalysisRunner( // IPC has the same semantics on macOS and Windows. The worker exits only // after reaching a JS-visible safe point; this parent keeps ownership until then. try { - child.send({ type: 'cancel' }); + child?.send({ type: 'cancel' }); } catch { // A closed IPC channel still has an exit/error path. Do not force-kill a // worker that may be inside native code. @@ -192,72 +219,121 @@ export function createAutoSyncAnalysisRunner( const onAbort = () => requestCancellation(new Error('Analysis cancelled.')); signal?.addEventListener('abort', onAbort, { once: true }); - child.on('message', (message: WorkerMessage) => { - // Once timeout/cancellation requested shutdown, its reason owns the - // result. A terminal IPC can already be queued behind cancellation. - if (message.type === 'graph-prepared' || terminalOutcome || terminationError) return; - if (message.type === 'progress') { - // The worker also sends its log lines as progress with percent -1; - // those are not progress. A listener's failure must not decide the - // run's outcome. - if (message.percent < 0) return; - try { - onProgress?.(message.percent, message.message); - } catch { - // ignored + const launch = (continuation?: ContinueMessage) => { + child = deps.forkWorker(workerPath, execArgv); + const phaseChild = child; + let prepared: GraphPreparedMessage | undefined; + child.stdout?.resume(); + child.stderr?.resume(); + child.on('message', (message: WorkerMessage) => { + // Once timeout/cancellation requested shutdown, its reason owns the + // result. A terminal IPC can already be queued behind cancellation. + if (terminalOutcome || terminationError || settled || child !== phaseChild) return; + if (message.type === 'progress') { + // Worker log messages use percent -1 and do not represent progress. + if (message.percent < 0) return; + try { + onProgress?.(message.percent, message.message); + } catch { + // A listener failure must not decide the run's outcome. + } + return; + } + if (message.type === 'graph-ready') { + // Watch has no serving reader cache. Publication is already verified. + try { + child.send({ type: 'graph-visible' }); + } catch (error) { + requestCancellation(error as Error); + } + return; + } + if (message.type === 'graph-prepared') { + try { + if (prepared || continuation) throw new Error('Repeated graph preparation'); + prepared = { ...message, handoff: projectPreparedAnalysisHandoff(message.handoff) }; + } catch (error) { + requestCancellation(error as Error); + } + return; + } + terminalOutcome = message; + }); + child.on('error', (error) => { + const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); + requestCancellation(workerError); + // This settles immediately rather than waiting out the grace, so the + // grace timer that would otherwise have released the child is cleared + // by cleanup(). Release it here instead — an errored channel does not + // mean the worker stopped. + releaseChild(); + settle(workerError); + }); + child.on('exit', (code, childSignal) => { + if (settled || child !== phaseChild) return; + if (terminationError) { + settle(terminationError); + return; + } + if (terminalOutcome?.type === 'complete') { + if (code === 0 && !childSignal) { + const { stats, graphWriteCollapsed } = terminalOutcome.result; + settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats }); + } else + settle( + new Error( + `Auto-sync analyze worker exited unsuccessfully (${childSignal ?? code}).`, + ), + ); + return; + } + if (!terminalOutcome && prepared && code === 0 && !childSignal && phaseChild.pid) { + try { + launch({ + type: 'continue', + repoPath, + options: analysisOptions, + writerPid: phaseChild.pid, + prepared, + }); + } catch (error) { + settle(error as Error); + } + return; + } + if (terminalOutcome?.type === 'error') { + settle( + new AutoSyncAnalysisError(terminalOutcome.message, { + code: terminalOutcome.code, + retryable: terminalOutcome.retryable, + }), + ); + return; } - return; - } - terminalOutcome = message; - deps.clearTimeoutFn(timeout); - }); - child.on('error', (error) => { - const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); - requestCancellation(workerError); - // This settles immediately rather than waiting out the grace, so the - // grace timer that would otherwise have released the child is cleared - // by cleanup(). Release it here instead — an errored channel does not - // mean the worker stopped. - releaseChild(); - settle(workerError); - }); - child.on('exit', (code, childSignal) => { - if (settled) return; - if (terminationError) { - settle(terminationError); - return; - } - if (terminalOutcome?.type === 'complete') { - const { stats, graphWriteCollapsed } = terminalOutcome.result; - // Only present on a collapsed write, so a healthy result keeps its shape. - settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats }); - return; - } - if (terminalOutcome?.type === 'error') { settle( - new AutoSyncAnalysisError(terminalOutcome.message, { - code: terminalOutcome.code, - retryable: terminalOutcome.retryable, - }), + new Error( + `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, + ), ); - return; + }); + try { + child.send(continuation ?? { type: 'start', repoPath, options: analysisOptions }); + } catch (error) { + const startError = new Error( + `Failed to start auto-sync analyze worker: ${(error as Error).message}`, + ); + requestCancellation(startError); + releaseChild(); + settle(startError); } - settle( - new Error( - `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, - ), - ); - }); + }; try { - child.send({ type: 'start', repoPath, options }); + launch(); } catch (error) { - const startError = new Error( - `Failed to start auto-sync analyze worker: ${(error as Error).message}`, - ); - requestCancellation(startError); - settle(startError); + settle(error as Error); } }); + }; } export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner(); diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index ba42d52a7..1f938ee38 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -248,6 +248,7 @@ export async function runEmbeddingJob( ? inputs.filter((input) => !complete.has(input.nodeId)) : inputs; if (missingInputs.length) await store.beginNodes(missingInputs); + const copied: Array<{ nodeId: string; contentHash: string; chunks: number }> = []; for (let i = 0; i < fetched.nodes.length; i++) { const node = fetched.nodes[i]; const input = inputs[i]; @@ -300,10 +301,7 @@ export async function runEmbeddingJob( release(); } } - complete.set( - node.id, - await store.finishNode(node.id, input.contentHash, reusable.chunks), - ); + copied.push({ nodeId: node.id, contentHash: input.contentHash, chunks: reusable.chunks }); reusedNodes++; continue; } @@ -374,7 +372,8 @@ export async function runEmbeddingJob( if (!abort) await flush(); const finished = [...generated] .filter(([id, node]) => node.ended && !failed.has(id)) - .map(([nodeId, node]) => ({ nodeId, contentHash: node.contentHash, chunks: node.chunks })); + .map(([nodeId, node]) => ({ nodeId, contentHash: node.contentHash, chunks: node.chunks })) + .concat(copied); if (finished.length) for (const receipt of await store.finishNodes(finished)) complete.set(receipt.nodeId, receipt); diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 7ec5a6f95..1f2a64e68 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -1283,7 +1283,14 @@ export const semanticSearch = async ( k: number = 10, maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), snapshot?: VectorSearchSnapshot, + embedQuery?: (text: string) => Promise, ): Promise => { + const queryVector = async (): Promise => { + if (embedQuery) return [...(await embedQuery(query))]; + if (!isEmbedderReady()) + throw new Error('Embedding model not initialized. Run embedding pipeline first.'); + return embeddingToArray(await embedText(query)); + }; let bestChunks = new Map< string, { distance: number; chunkIndex: number; startLine: number; endLine: number } @@ -1291,16 +1298,7 @@ export const semanticSearch = async ( let generationSelected = false; if (snapshot) { const { searchVectorGeneration } = await import('./vector-generation-search.js'); - const chunks = await searchVectorGeneration( - snapshot, - async () => { - if (!isEmbedderReady()) - throw new Error('Embedding model not initialized. Run embedding pipeline first.'); - return embeddingToArray(await embedText(query)); - }, - k, - maxDistance, - ); + const chunks = await searchVectorGeneration(snapshot, queryVector, k, maxDistance); generationSelected = chunks !== null; if (chunks) bestChunks = new Map(chunks.map(({ nodeId, ...chunk }) => [nodeId, chunk])); } @@ -1312,12 +1310,7 @@ export const semanticSearch = async ( return []; } - if (!isEmbedderReady()) { - throw new Error('Embedding model not initialized. Run embedding pipeline first.'); - } - - const queryEmbedding = await embedText(query); - const queryVec = embeddingToArray(queryEmbedding); + const queryVec = await queryVector(); const queryVecStr = `[${queryVec.join(',')}]`; // Query/read path: NEVER spawn a network INSTALL on a user query. If the diff --git a/gitnexus/src/core/embeddings/vector-generation-search.ts b/gitnexus/src/core/embeddings/vector-generation-search.ts index dc09f7464..6f028a0b7 100644 --- a/gitnexus/src/core/embeddings/vector-generation-search.ts +++ b/gitnexus/src/core/embeddings/vector-generation-search.ts @@ -110,7 +110,7 @@ export async function searchVectorGeneration( } } if (selected.generation.chunks > exactLimit) return []; - // LIMIT is a defensive bound against edited metadata, not pagination. + // determinism: probe — fetch the entire verified set; reject partial or oversized coverage before ranking. const rows = await execute(`MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding LIMIT ${exactLimit + 1}`); diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 40f7b4933..dcd5bf424 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -2444,7 +2444,9 @@ export const fetchExistingEmbeddingHashes = async ( */ export const flushWAL = async (): Promise => { const c = conn; - if (!c) return; + // CHECKPOINT on a read-only connection can leave LadybugDB's checkpoint + // marker set even though the write is rejected, poisoning the next reader. + if (!c || currentDbReadOnly) return; try { await withConnLock(async () => { const checkpointResult = await c.query('CHECKPOINT'); @@ -2473,7 +2475,7 @@ export const flushWAL = async (): Promise => { */ export const tryFlushWAL = async (): Promise => { const c = conn; - if (!c) return false; + if (!c || currentDbReadOnly) return false; // Runs on the periodic WAL-checkpoint driver. The lock makes this CHECKPOINT // wait for any in-flight COPY / writeback on the singleton connection instead // of executing concurrently with it (the `analyze --pdg` heap-corruption bug). diff --git a/gitnexus/src/core/lbug/legacy-vector-export.ts b/gitnexus/src/core/lbug/legacy-vector-export.ts new file mode 100644 index 000000000..52db9bd4a --- /dev/null +++ b/gitnexus/src/core/lbug/legacy-vector-export.ts @@ -0,0 +1,177 @@ +/** Disk-backed migration of the legacy vector table, without copying its graph. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import lbug from '@ladybugdb/core'; +import { openLbugConnection, closeLbugConnection } from './lbug-config.js'; +import { escapeCypherString } from './cypher-escape.js'; +import { assertLbugPublicationReady } from './publication.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; +import { + recordLegacyEmbeddingSource, + type LegacyEmbeddingSourceRecord, +} from '../../storage/legacy-embedding-source.js'; + +export interface LegacyVectorExportOptions { + metadataDirectory: string; + /** The caller owns the slot lock and a stable source connection. Only catalog, + * count and native COPY results cross this callback, never vector payloads. */ + query: (cypher: string) => Promise; + provenance: Pick< + LegacyEmbeddingSourceRecord, + 'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity' + >; + excluded: Iterable | AsyncIterable; + /** Additional native pool, admitted alongside the source's pool. */ + bufferPoolBytes: number; + threads: number; + signal?: AbortSignal; +} + +const fields = ['id', 'nodeId', 'chunkIndex', 'startLine', 'endLine', 'embedding', 'contentHash']; +const countQuery = 'MATCH (e:CodeEmbedding) RETURN count(e) AS count'; + +/** Preserve the old physical width before writable schema initialization can + * replace its table. The caller publishes the returned receipt under its lock. */ +export async function exportLegacyVectorsFromFile( + sourcePath: string, + options: Omit & { sourcePoolBytes: number }, +): Promise { + options.signal?.throwIfAborted(); + const stat = await fs.lstat(sourcePath); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy vector source must be an owned regular file'); + const handle = await openLbugConnection(lbug, sourcePath, { + readOnly: true, + bufferPoolBytes: options.sourcePoolBytes, + threads: options.threads, + }); + try { + return await exportLegacyVectors({ + ...options, + query: async (sql) => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected legacy vector export result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }, + }); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: sourcePath }); + } +} + +function count(rows: unknown[]): number { + const value = (rows[0] as { count?: unknown } | undefined)?.count; + if (rows.length !== 1 || (typeof value !== 'number' && typeof value !== 'bigint')) + throw new Error('Legacy vector export count is unverified'); + const n = Number(value); + if (!Number.isSafeInteger(n) || n < 0) throw new Error('Legacy vector export count is invalid'); + return n; +} + +/** The returned receipt pins an independent, immutable table-only database. + * Group completeness/input compatibility is still checked by LegacyVectorSource; + * copying a row does not certify its historical provider or text configuration. + * The source and its metadata are never changed here. Publish the returned + * reference under the slot lock before any source mutation or cleanup. */ +export async function exportLegacyVectors( + options: LegacyVectorExportOptions, +): Promise { + options.signal?.throwIfAborted(); + if ( + !Number.isSafeInteger(options.bufferPoolBytes) || + options.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(options.threads) || + options.threads < 1 + ) + throw new Error('Legacy vector export requires an admitted native pool'); + const root = await embeddingJobDirectory(options.metadataDirectory); + const expected = count(await options.query(countQuery)); + if (expected === 0) return undefined; + const catalog = (await options.query("CALL table_info('CodeEmbedding') RETURN *")) as { + name?: unknown; + type?: unknown; + 'primary key'?: unknown; + }[]; + const vectorType = catalog.find((column) => column.name === 'embedding')?.type; + const match = typeof vectorType === 'string' && /^FLOAT\[([1-9]\d*)\]$/.exec(vectorType); + const dimensions = match ? Number(match[1]) : NaN; + const types = ['STRING', 'STRING', 'INT32', 'INT64', 'INT64', vectorType, 'STRING']; + if ( + !Number.isSafeInteger(dimensions) || + dimensions < 1 || + fields.some((name, i) => !catalog.some((c) => c.name === name && c.type === types[i])) || + !catalog.some((c) => c.name === 'id' && c['primary key'] === true) || + (options.provenance.recordedIdentity?.dimensions !== undefined && + options.provenance.recordedIdentity.dimensions !== dimensions) + ) + throw new Error('Legacy vector schema differs from its recorded provenance; source retained'); + + const id = randomUUID(); + const staging = path.join(root, `lbug.staging.${id}`); + const csv = staging + '.csv'; + const sourceFile = `lbug.generation.${id.replaceAll('-', '')}`; + const destination = path.join(root, sourceFile); + const csvLiteral = `'${escapeCypherString(csv)}'`; + try { + options.signal?.throwIfAborted(); + await options.query( + `COPY (MATCH (e:CodeEmbedding) RETURN ${fields.map((f) => `e.${f} AS ${f}`).join(', ')}) TO ${csvLiteral} (HEADER=true)`, + ); + options.signal?.throwIfAborted(); + const handle = await openLbugConnection(lbug, staging, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }); + const query = async (sql: string): Promise => { + const result = await handle.conn.query(sql); + const results = Array.isArray(result) ? result : [result]; + try { + if (results.length !== 1) throw new Error('Unexpected legacy vector export result'); + return await results[0].getAll(); + } finally { + for (const cursor of results) cursor.close(); + } + }; + try { + await query( + `CREATE NODE TABLE CodeEmbedding (${fields.map((name, i) => `${name} ${types[i]}`).join(', ')}, PRIMARY KEY(id))`, + ); + await query(`COPY CodeEmbedding FROM ${csvLiteral} (HEADER=true, auto_detect=false)`); + if ( + count(await query(countQuery)) !== expected || + count(await options.query(countQuery)) !== expected + ) + throw new Error('Legacy vector export lost rows or its source changed; source retained'); + options.signal?.throwIfAborted(); + await query('CHECKPOINT'); + } finally { + // A failed physical release remains charged by the native reservation + // manager. Never rename or certify that still-open export. + await closeLbugConnection(handle, { strict: true, databasePath: staging }); + } + options.signal?.throwIfAborted(); + await assertLbugPublicationReady(staging); + await fs.rename(staging, destination); + const file = await fs.open(destination, 'r+'); + try { + await file.sync(); + } finally { + await file.close(); + } + await syncEmbeddingJobDirectory(root); + return await recordLegacyEmbeddingSource( + root, + { sourceFile, ...options.provenance }, + options.excluded, + options.signal, + ); + } finally { + await fs.rm(csv, { force: true }); + } +} diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 1a13aa737..467f5ae65 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -503,32 +503,39 @@ export class VectorStore { }, true); } - private async verify(receipt: VectorNodeReceipt): Promise { - for (let start = 0; start < receipt.chunks; start += BLOCK_ROWS) { - const ids = Array.from( - { length: Math.min(BLOCK_ROWS, receipt.chunks - start) }, - (_, i) => `${receipt.nodeId}:${start + i}`, - ); + private async verify(input: VectorNodeReceipt | readonly VectorNodeReceipt[]): Promise { + const receipts = 'nodeId' in input ? [input] : input; + const expected = new Map(); + const flush = async () => { + if (!expected.size) return; const rows = await this.query( 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash', - { ids }, + { ids: [...expected.keys()] }, ); const seen = new Set(); for (const row of rows) { - const i = count(row.chunkIndex); + const entry = typeof row.id === 'string' ? expected.get(row.id) : undefined; if ( - row.nodeId !== receipt.nodeId || - row.contentHash !== receipt.contentHash || - i < start || - i >= start + ids.length || - row.id !== `${receipt.nodeId}:${i}` || + !entry || + row.nodeId !== entry.receipt.nodeId || + row.contentHash !== entry.receipt.contentHash || + count(row.chunkIndex) !== entry.ordinal || seen.has(String(row.id)) ) throw new Error('Vector node has an invalid chunk set'); seen.add(String(row.id)); } - if (seen.size !== ids.length) throw new Error('Vector node is missing chunks'); - } + if (seen.size !== expected.size) throw new Error('Vector node is missing chunks'); + expected.clear(); + }; + // Most groups contain one chunk. Fill each bounded PK query across groups + // rather than paying a native round trip for every node in a unit. + for (const receipt of receipts) + for (let ordinal = 0; ordinal < receipt.chunks; ordinal++) { + expected.set(`${receipt.nodeId}:${ordinal}`, { receipt, ordinal }); + if (expected.size === BLOCK_ROWS) await flush(); + } + await flush(); } async finishNode( @@ -560,9 +567,9 @@ export class VectorStore { node.chunks < 1 ) throw new Error('Vector node coverage does not match its generated chunks'); - await this.verify(receipt); verified.push(receipt); } + await this.verify(verified); await this.query( 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', { ids: nodes.map((n) => n.nodeId) }, @@ -868,6 +875,7 @@ export class VectorStore { } private async verifyVectorIndex(): Promise { + // determinism: probe — any stored vector can verify index availability; its identity does not affect publication. const sample = await this.query( 'MATCH (e:CodeEmbedding) RETURN e.embedding AS embedding LIMIT 1', ); diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index da443264c..227ecfbbf 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -38,15 +38,12 @@ import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js'; import path from 'path'; import fs from 'fs/promises'; import { constants as fsConstants, existsSync } from 'node:fs'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import { retryRename } from '../storage/fs-atomic.js'; import { assertLbugPublicationReady, consolidateLbugForPublication } from './lbug/publication.js'; -import { - acquireIndexLock, - requireExclusiveIndexLock, - sweepStagingArtifacts, -} from '../storage/index-lock.js'; +import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { resolveEmbeddingRecovery } from '../storage/embedding-recovery.js'; import { invalidateNodeWorkspacePackages } from './ingestion/languages/typescript/workspace-packages.js'; import { @@ -65,7 +62,11 @@ import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/und import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import { summarizeUnresolvedOverloads } from './ingestion/scope-resolution/unresolved-overloads.js'; import type { KnowledgeGraph } from './graph/types.js'; -import type { PreparedAnalysisHandoff } from './analysis-phase-supervisor.js'; +import { + publishAnalysisAfterProcessExit, + type AnalysisPhaseTarget, + type PreparedAnalysisHandoff, +} from './analysis-phase-supervisor.js'; import { reconcileGraphNodeIdentities } from './incremental/write-reconciliation.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { @@ -176,6 +177,8 @@ import { loadMeta, ensureGitNexusIgnored, registerRepo, + readRegistry, + findRegistryEntryByRepoPath, setShareOptOut, adoptFlatBranchLabel, isReadOnlyFilesystemError, @@ -216,7 +219,17 @@ import { resolveOptedInStore, seedSharedSlot, } from './shared-store-analyze.js'; -import { publishAnalysisParseCache } from './analysis-parse-cache.js'; +import { + publishAnalysisParseCache, + loadAnalysisParseCache, + stageAnalysisParseCache, +} from './analysis-parse-cache.js'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from './analysis-publication.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js'; import { DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, @@ -456,6 +469,12 @@ function collectSpringBeanDeclarationDriftFiles( // --------------------------------------------------------------------------- export interface AnalyzeCallbacks { + /** Process-local cancellation; never serialized through options/IPC. */ + signal?: AbortSignal; + /** Exit-only frontends admit their supervisor before opening a graph writer. */ + onGraphWork?: () => void | Promise; + /** Let a serving parent refresh graph readers before paid embedding work. */ + onGraphPublished?: () => Promise; onProgress: ( phase: string, percent: number, @@ -466,6 +485,9 @@ export interface AnalyzeCallbacks { } export interface AnalyzeOptions { + /** Internal phase coordination. Entry points activate this together with + * the published embedding continuation; never a user-facing mode switch. */ + graphPublication?: 'auto' | 'inline' | 'after-exit'; /** Frozen per-invocation allocations; entrypoints resolve these before destructive work. */ resources?: import('./analysis-resource-budget.js').AnalysisResourceBudget; /** @@ -1446,6 +1468,237 @@ interface ParseIdentityRun { storagePath?: string; } +async function retainsPreparedGraph(metadataDirectory: string): Promise { + return fs.lstat(path.join(metadataDirectory, 'analysis-publication.json')).then( + () => true, + (error: NodeJS.ErrnoException) => error.code !== 'ENOENT', + ); +} + +/** Preview the registry's existing name precedence without advancing freshness. + * The continuation checks it again under the slot lock before publication. */ +async function preparedAnalysisName(repoPath: string, options: AnalyzeOptions): Promise { + if (options.registryName !== undefined) return options.registryName; + const existing = findRegistryEntryByRepoPath(await readRegistry(), repoPath); + const inferred = getInferredRepoName(repoPath); + if ( + existing && + existing.name !== path.basename(path.resolve(existing.path)) && + existing.name !== path.basename(resolveRepoIdentityRoot(existing.path)) && + existing.name !== inferred + ) + return existing.name; + return inferred ?? path.basename(resolveRepoIdentityRoot(repoPath)); +} + +async function recoverPreparedGraph( + target: WriteTarget, + repoPath: string, + options: AnalyzeOptions, + log: (message: string) => void, +): Promise { + const prepared = await retainsPreparedGraph(target.metaDir); + if (prepared && !(await readPreparedAnalysis(target.metaDir))) return undefined; + // Publication removes the graph receipt before the optional cache is merged. + // Recover that crash window using only the currently published generation. + const alreadyPublished = prepared ? undefined : await loadMeta(target.metaDir); + let pendingCache: Awaited>; + if (!prepared) { + if (!alreadyPublished?.graphGeneration || alreadyPublished.incrementalInProgress) + return undefined; + try { + pendingCache = await loadAnalysisParseCache(target.metaDir, alreadyPublished); + } catch (error) { + log(`Warning: could not recover the prepared parse cache (${(error as Error).message}).`); + return undefined; + } + if (!pendingCache) return undefined; + } + if (!options.resources) throw new Error('Graph publication requires admitted resources'); + const metadata = + alreadyPublished ?? + (await publishPreparedAnalysis(target.metaDir, { + saveMetadata: (meta) => saveMeta(target.metaDir, meta), + verify: async (file, meta) => { + if (meta.repoPath !== repoPath) + throw new Error('Prepared graph belongs to another checkout'); + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + await verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: options.resources!.parse.graphPoolBytes, + threads: options.resources!.parse.nativeThreads, + }); + }, + })); + if (!metadata) throw new Error('Prepared graph disappeared under its writer lock'); + await registerRepo(repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: target.placement.branch, + storagePath: target.storagePath, + }); + try { + const cache = pendingCache ?? (await loadAnalysisParseCache(target.metaDir, metadata)); + if (cache) + await publishAnalysisParseCache(cache, { + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + cacheRoot: target.sharedStore?.cachesDir ?? target.storagePath, + publishedGeneration: metadata.graphGeneration?.id, + sharedStore: target.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + } catch (error) { + log( + `Warning: could not publish the prepared parse cache (${(error as Error).message}); continuing.`, + ); + } + return metadata; +} + +/** Complete an exit-only writer's exact attempt, without invoking the parser or + * regenerating context/skills. The launcher must wait for actual process exit + * before starting this continuation in its separately admitted process. */ +export async function continuePreparedAnalysis( + handoff: PreparedAnalysisHandoff, + writerPid: number, + options: AnalyzeOptions, + callbacks: AnalyzeCallbacks, + signal?: AbortSignal, +): Promise { + const resources = options.resources; + if (!resources) throw new Error('Graph continuation requires admitted resources'); + const log = (message: string) => callbacks.onLog?.(stripControlCharacters(message)); + let target = await resolveWriteTarget(handoff.repoPath, options); + const resolveTarget = async (): Promise => { + target = await resolveWriteTarget(handoff.repoPath, options); + return { + repoPath: handoff.repoPath, + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + repoName: await preparedAnalysisName(handoff.repoPath, options), + commit: target.currentCommit, + }; + }; + const finalize = async (metadata: RepoMeta): Promise => { + await registerRepo(handoff.repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: target.placement.branch, + storagePath: target.storagePath, + }); + if (!target.placement.branch && target.branchLabel) { + try { + await adoptFlatBranchLabel(handoff.repoPath, target.branchLabel, target.storagePath); + } catch (error) { + log(`Warning: could not sync the workspace branch label (${(error as Error).message}).`); + } + } + if (target.sharedStore && !target.placement.branch) { + await publishSharedGraph(target.sharedStore, handoff.repoPath, target.currentCommit, log, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + }); + } else if (!target.sharedStore) { + const prior = options.noShare ? await optedInSlotToLeave(handoff.repoPath) : undefined; + if (prior) await leaveSharedStore(handoff.repoPath, prior, target.storagePath, log); + else await registerLeftStore(handoff.repoPath, target.storagePath); + } + if (options.noShare) await setShareOptOut(handoff.repoPath, true); + else if (options.shareWith) await setShareOptOut(handoff.repoPath, false); + return (await loadMeta(target.metaDir)) ?? metadata; + }; + let metadata = await publishAnalysisAfterProcessExit(handoff, writerPid, { + resolveTarget, + saveMetadata: (meta) => saveMeta(target.metaDir, meta), + verify: async (file, meta) => { + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + await verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }); + }, + finalizeGraph: async (meta) => { + const published = await finalize(meta); + try { + const cache = await loadAnalysisParseCache(target.metaDir, published); + if (cache) + await publishAnalysisParseCache(cache, { + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + cacheRoot: target.sharedStore?.cachesDir ?? target.storagePath, + publishedGeneration: published.graphGeneration?.id, + sharedStore: target.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + await removeColdParseRebuildDir(getColdParseRebuildDir(target.metaDir), false); + } catch (error) { + log(`Warning: could not publish the prepared parse cache (${(error as Error).message}).`); + } + callbacks.onProgress('graph-ready', 90, 'Graph published', { graphPhase: 'ready' }); + if ( + !_deriveEmbeddingMode(options, published.stats?.embeddings ?? 0).shouldGenerateEmbeddings && + !published.embeddingJob && + !published.embeddingCheckpoint + ) + await callbacks.onGraphPublished?.(); + return published; + }, + signal, + log, + onWaitStart: () => callbacks.onProgress('lock', 0, 'Waiting to publish the prepared graph…'), + }); + const generate = _deriveEmbeddingMode( + options, + metadata.stats?.embeddings ?? 0, + ).shouldGenerateEmbeddings; + if (generate || metadata.embeddingJob || metadata.embeddingCheckpoint) { + signal?.throwIfAborted(); + const lock = await acquireAnalysisSlotLock(handoff.metadataDirectory, { sweep: false, log }); + try { + requireExclusiveIndexLock(lock, 'Embedding continuation requires exclusive index ownership'); + const current = await resolveTarget(); + for (const key of [ + 'repoPath', + 'storagePath', + 'metadataDirectory', + 'repoName', + 'commit', + ] as const) + if (current[key] !== handoff[key]) + throw new Error('Analysis target changed before embedding; retry analysis.'); + const { runAnalysisEmbeddingPhase } = await import('./analysis-embedding-phase.js'); + metadata = await runAnalysisEmbeddingPhase({ + metadataDirectory: target.metaDir, + graphPath: resolveGraphPath(target.metaDir), + resources, + callbacks, + nodeLimit: options.embeddingsNodeLimit, + expectedGeneration: handoff.publicationId, + signal, + }); + metadata = await finalize(metadata); + // The graph writer deliberately retained staging files across its exit. + // Reclaim them only after the replacement vectors are published, while + // still holding the ordinary slot lock. + sweepStagingArtifacts(target.metaDir, log); + } finally { + lock.release(); + } + } + callbacks.onProgress('done', 100, 'Done', { graphPhase: 'ready' }); + return metadata; +} + function effectiveParseCacheIdentity( identity: ParseCacheIdentity, options: Pick, @@ -1483,6 +1736,12 @@ export async function runFullAnalysis( callbacks: AnalyzeCallbacks, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { + if (options.graphPublication && !options.resources) + throw new Error('Graph publication requires admitted resources'); + if (options.graphPublication === 'after-exit' && !options.skipNativeCloseOnExit) + throw new Error('Deferred publication requires an exit-only graph writer'); + if (options.graphPublication === 'inline' && options.skipNativeCloseOnExit) + throw new Error('Inline publication requires physical native release'); // Validate operator-provided FTS config before anything else — a typo fails // here in ms, without taking the lock. (createSearchFTSIndexes reuses the // cached value via getSearchFTSStemmer.) @@ -1514,7 +1773,8 @@ export async function runFullAnalysis( }; let writeTarget = await resolveWriteTarget(repoPath, options); - let lock = await acquireIndexLock(writeTarget.metaDir, acquireOpts); + let lock = await acquireAnalysisSlotLock(writeTarget.metaDir, acquireOpts); + let retainParseStage = false; return withEmbeddingSpillScope(async () => { try { requireExclusiveIndexLock( @@ -1549,7 +1809,7 @@ export async function runFullAnalysis( ); lock.release(); writeTarget = fresh; - lock = await acquireIndexLock(fresh.metaDir, acquireOpts); + lock = await acquireAnalysisSlotLock(fresh.metaDir, acquireOpts); requireExclusiveIndexLock( lock, `Cannot acquire the index lock at ${fresh.metaDir}; refusing an unlocked analysis.`, @@ -1565,10 +1825,35 @@ export async function runFullAnalysis( `Warning: checkout "${formatRejectedBranchForLog(writeTarget.rejectedDetectedBranch)}" is not a usable index label; continuing.`, ); } + if (options.graphPublication === 'auto') { + const prior = await loadMeta(writeTarget.metaDir); + const separate = Boolean( + prior?.graphGeneration || + prior?.embeddingJob || + (!options.dropEmbeddings && prior?.embeddingCheckpoint) || + _deriveEmbeddingMode(options, prior?.stats?.embeddings ?? 0).shouldGenerateEmbeddings, + ); + options = { + ...options, + graphPublication: separate + ? options.skipNativeCloseOnExit + ? 'after-exit' + : 'inline' + : undefined, + }; + } // Only this canonical slot is locked. Its deterministic parser stage // can only belong to an interrupted predecessor; sibling stages live // under their own meta directories and must remain untouched. Reclaim // before freshness/FTS-only early returns as well as parsing runs. + if (options.graphPublication) { + const recovered = await recoverPreparedGraph(writeTarget, repoPath, options, log); + if (recovered?.graphPath) + writeTarget = { + ...writeTarget, + lbugPath: path.join(writeTarget.metaDir, recovered.graphPath), + }; + } await removeColdParseRebuildDir(getColdParseRebuildDir(writeTarget.metaDir), false); const flatShared = writeTarget.placement.branch ? undefined : writeTarget.sharedStore; if (flatShared) { @@ -1599,6 +1884,10 @@ export async function runFullAnalysis( parseRun, runnerIdentityAtBootstrap, ); + if (result.preparedGraph) { + retainParseStage = true; + return result; + } if (flatShared) { await publishSharedGraph(flatShared, repoPath, writeTarget.currentCommit, log, { name: options.registryName, @@ -1619,11 +1908,78 @@ export async function runFullAnalysis( // A clone that left stays out of sibling stores until `--share-with`. if (options.noShare) await setShareOptOut(repoPath, true); else if (options.shareWith) await setShareOptOut(repoPath, false); + if ( + options.graphPublication === 'inline' || + (options.graphPublication === 'after-exit' && result.alreadyUpToDate) + ) { + const published = await loadMeta(writeTarget.metaDir); + const generate = _deriveEmbeddingMode( + options, + published?.stats?.embeddings ?? 0, + ).shouldGenerateEmbeddings; + if (published && (generate || published.embeddingJob || published.embeddingCheckpoint)) { + const { runAnalysisEmbeddingPhase } = await import('./analysis-embedding-phase.js'); + const metadata = await runAnalysisEmbeddingPhase({ + metadataDirectory: writeTarget.metaDir, + graphPath: path.resolve(writeTarget.metaDir, published.graphPath ?? 'lbug'), + resources: options.resources!, + callbacks, + nodeLimit: options.embeddingsNodeLimit, + expectedGeneration: published.graphGeneration?.id, + signal: callbacks.signal, + }); + // A matching completed artifact performs no writes. Preserve the + // graph's no-op result and avoid republishing identical registry and + // shared-store metadata on every cached embedding invocation. + const unchangedVectors = + result.alreadyUpToDate && + !published.embeddingJob && + !published.embeddingCheckpoint && + published.vectorGeneration && + metadata.vectorGeneration?.identity === published.vectorGeneration.identity; + if (!unchangedVectors) { + await registerRepo(repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: writeTarget.placement.branch, + storagePath: writeTarget.storagePath, + }); + if (flatShared) + await publishSharedGraph(flatShared, repoPath, writeTarget.currentCommit, log, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + }); + result.alreadyUpToDate = false; + } + result.stats = metadata.stats ?? {}; + callbacks.onProgress('done', 100, 'Done', { + graphPhase: 'ready', + embeddingProgress: { + phase: + metadata.vectorGeneration?.graphGeneration === metadata.graphGeneration?.id && + !metadata.embeddingJob && + metadata.vectorGeneration + ? 'ready' + : metadata.stats?.embeddings + ? 'preserved' + : 'unverified', + }, + }); + } else { + await callbacks.onGraphPublished?.(); + } + } return result; } finally { // Readers opened after analyze must not inherit its smaller write budget. setBufferPoolSizeHint(undefined); - await removeColdParseRebuildDir(parseRun.storagePath, true); + if (options.graphPublication && !retainParseStage) { + // A close/publication failure can occur after the durable handoff was + // written but before an AnalyzeResult is returned. Preserve its cache + // even when the receipt itself is unreadable. + retainParseStage = await retainsPreparedGraph(writeTarget.metaDir); + } + if (!retainParseStage) await removeColdParseRebuildDir(parseRun.storagePath, true); discardScopedEmbeddingSpills(); lock.release(); } @@ -1698,16 +2054,20 @@ async function runFullAnalysisInner( runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { const ftsDisabledReason = resolveFtsDisableReason(options.skipFts); - const initAnalysisLbug = (dbPath: string) => - options.resources - ? initLbug(dbPath, { - skipFts: !!ftsDisabledReason, - bufferPoolBytes: options.resources.parse.graphPoolBytes, - threads: options.resources.parse.nativeThreads, - }) - : ftsDisabledReason - ? initLbug(dbPath, { skipFts: true }) - : initLbug(dbPath); + const initAnalysisLbug = (dbPath: string) => { + if (options.resources) { + // Admission is a ceiling. Preserve the existing graph-sized cache so + // small repositories do not pay for a host-sized native allocation. + const sizedPool = getEffectiveBufferPoolSize(); + const ceiling = options.resources.parse.graphPoolBytes; + return initLbug(dbPath, { + skipFts: !!ftsDisabledReason, + bufferPoolBytes: Math.min(sizedPool || ceiling, ceiling), + threads: options.resources.parse.nativeThreads, + }); + } + return ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath); + }; const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); @@ -1782,12 +2142,16 @@ async function runFullAnalysisInner( }; const ftsProfile = ftsProfileForContentRetention(contentRetention); const ftsIndexes = getFtsIndexes(ftsProfile); + let graphResourcesAdmitted = false; const admitGraphResources = async (): Promise => { + if (graphResourcesAdmitted) return; + if (options.graphPublication === 'after-exit') await callbacks.onGraphWork?.(); if (!options.resources) return; const { drainEmbeddingSidecarForAnalysis } = await import('./embeddings/embedding-sidecar-client.js'); await drainEmbeddingSidecarForAnalysis(); await drainAnalysisNativeResources(); + graphResourcesAdmitted = true; }; // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open @@ -1817,6 +2181,7 @@ async function runFullAnalysisInner( // first graph open: here for the paths that open it before the up-to-date // check, and below once that check falls through. const ensurePrivateGraph = async (copy = true): Promise => { + await admitGraphResources(); if (!writeTarget.sharedStore || placement.branch) return; if (!(await ensurePrivateSharedGraph(metaDir, log, { copy }))) { collector.add({ @@ -2584,7 +2949,13 @@ async function runFullAnalysisInner( // it means an index predating the field, whose width is unknown but was // consistent with the env that wrote it, and which the fingerprint guard // above already rebuilds — that rebuild is where the stamp lands. - if (existingMeta && embeddingDimsMismatch(existingMeta.embeddingDims, EMBEDDING_DIMS)) { + // Published generations keep vectors in their own typed artifact. Changing + // that artifact's width must not invalidate the independently versioned graph. + if ( + existingMeta && + !(options.graphPublication && existingMeta.graphGeneration) && + embeddingDimsMismatch(existingMeta.embeddingDims, EMBEDDING_DIMS) + ) { // Only NAME a recorded width that could be one, for the reason the // fingerprint guard gates its stamp on `isSchemaFingerprintShaped`: // meta.json is a schema-less JSON.parse of on-disk state, so a value that @@ -2736,7 +3107,10 @@ async function runFullAnalysisInner( JSON.stringify(getIncludeDotDirs()); if ( existingMeta && - !existingMeta.embeddingCheckpoint && + (!existingMeta.embeddingCheckpoint || + (options.graphPublication && + resumeEmbeddingCheckpoint && + !existingMeta.embeddingCheckpoint.recovery)) && !options.force && existingMeta.lastCommit === currentCommit && !ftsModeChanged && @@ -2981,8 +3355,8 @@ async function runFullAnalysisInner( // proven in-place path; enable it only to test the Windows swap. const windowsSwapOk = process.platform === 'win32' && - options.pdg !== true && - process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1'; + (options.graphPublication !== undefined || + (options.pdg !== true && process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1')); // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: @@ -3021,7 +3395,16 @@ async function runFullAnalysisInner( shouldLoadCache: derivedShouldLoadCache, } = _deriveEmbeddingMode(options, existingEmbeddingCount); const shouldGenerateEmbeddings = derivedShouldGenerateEmbeddings || resumeEmbeddingCheckpoint; - const shouldLoadCache = derivedShouldLoadCache || resumeEmbeddingCheckpoint; + const shouldLoadCache = + !(options.graphPublication && (existingMeta?.vectorGeneration || existingMeta?.embeddingJob)) && + (derivedShouldLoadCache || resumeEmbeddingCheckpoint); + + const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint); + const stagedCheckpoint = existingMeta?.embeddingCheckpoint; + const inheritedUnsafeNodeIds = new Set([ + ...pendingEmbeddingNodeIds, + ...(stagedRecovery?.unsafeNodeIds ?? []), + ]); if (options.dropEmbeddings && existingEmbeddingCount > 0) { log( @@ -3057,6 +3440,42 @@ async function runFullAnalysisInner( // possible the dirty block above already threw a LbugWipeError, so this // open is replay-free by construction (FIX 1 of this shipping review). if (shouldLoadCache && existingMeta) { + if ( + options.graphPublication && + !options.dropEmbeddings && + !existingMeta.graphGeneration && + !existingMeta.legacyVectorSource && + // A supervisor handoff may leave only placeholder metadata on a first + // run. Missing storage is an error only when it claims paid vectors. + (existingEmbeddingCount > 0 || existsSync(lbugPath)) + ) { + const { exportLegacyVectorsFromFile } = await import('./lbug/legacy-vector-export.js'); + const historical = existingMeta.embeddingCheckpoint; + const legacy = await exportLegacyVectorsFromFile(lbugPath, { + metadataDirectory: metaDir, + provenance: { + schemaFingerprint: existingMeta.schemaFingerprint ?? SCHEMA_FINGERPRINT, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: inheritedUnsafeNodeIds, + sourcePoolBytes: options.resources!.embedding.graphPoolBytes, + bufferPoolBytes: options.resources!.embedding.vectorPoolBytes, + threads: options.resources!.embedding.nativeThreads, + signal: callbacks.signal, + }); + if (legacy) { + await saveMeta(metaDir, { ...existingMeta, legacyVectorSource: legacy }); + existingMeta.legacyVectorSource = legacy; + } + } // Cache initialization is writable and may discard quarantined WALs. // Preserve the live family before this first open, including runs whose // incremental write later escalates to a staged full rebuild. Keep this @@ -3089,12 +3508,6 @@ async function runFullAnalysisInner( // A checkpoint's pending decision and its paid, complete vectors are // independent: --force discards the former but can still reuse the latter. // Select only the explicitly referenced generation, never an orphan by age. - const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint); - const stagedCheckpoint = existingMeta?.embeddingCheckpoint; - const inheritedUnsafeNodeIds = new Set([ - ...pendingEmbeddingNodeIds, - ...(stagedRecovery?.unsafeNodeIds ?? []), - ]); if (shouldLoadCache && !options.dropEmbeddings && stagedRecovery && stagedCheckpoint) { if (!embeddingIdentityForRun) { const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js'); @@ -3484,6 +3897,33 @@ async function runFullAnalysisInner( // orphan sweep (sweepStagingArtifacts) reclaims stragglers on the next // acquire. The `.staging.` prefix is what that sweep matches. let buildPath = useAtomicSwap ? `${lbugPath}.staging.${randomUUID()}` : lbugPath; + let inPlaceAttempt: string | undefined; + let inPlaceVectorCoverage: RepoMeta['vectorGeneration']; + const saveGraphDirtyState = async (metadata: RepoMeta): Promise => { + if (!options.graphPublication || buildPath !== lbugPath) { + await saveMeta(metaDir, metadata); + return; + } + if (!inPlaceAttempt) { + const dirty = await beginInPlaceAnalysis(metaDir, metadata, (meta) => + saveMeta(metaDir, meta), + ); + inPlaceAttempt = dirty.incrementalInProgress?.attemptId; + inPlaceVectorCoverage = dirty.vectorGeneration; + return; + } + if (!metadata.incrementalInProgress) throw new Error('Missing in-place write state'); + await saveMeta(metaDir, { + ...metadata, + graphGeneration: undefined, + vectorGeneration: inPlaceVectorCoverage, + incrementalInProgress: { + ...metadata.incrementalInProgress, + attemptId: inPlaceAttempt, + writePlan: 'in-place', + }, + }); + }; if (isIncremental && hashDiff) { log( @@ -3500,7 +3940,7 @@ async function runFullAnalysisInner( // force-rebuild a healthy index after a crash before swap. if (!atomicIncremental) { const now = Date.now(); - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta!, incrementalInProgress: { startedAt: now, @@ -3535,9 +3975,9 @@ async function runFullAnalysisInner( // clean-tree run's fast path would certify a destroyed DB (or, after a // pdg flip, certify zombie/missing BasicBlock rows indefinitely). // toWriteCount: 0 is the full-path sentinel (no incremental write set). - if (existingMeta) { + if (existingMeta && (!options.graphPublication || !useAtomicSwap)) { const now = Date.now(); - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta, incrementalInProgress: { startedAt: now, @@ -3740,7 +4180,7 @@ async function runFullAnalysisInner( ): Promise => { // Do not stamp live metadata while writing a staging database. if (buildPath !== lbugPath) return; - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta!, incrementalInProgress: { startedAt: dirtyStartedAt, @@ -4508,7 +4948,7 @@ async function runFullAnalysisInner( // shipped defaults is a match, so a budget-mismatch derived rewrite that // dies in FTS would otherwise recertify the rewritten Community/Process // rows on a flagless retry. - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...base, incrementalInProgress: buildFtsDirtyStamp({ prior: base.incrementalInProgress, @@ -5037,7 +5477,7 @@ async function runFullAnalysisInner( } let stagedCheckpointEmbeddingCount: number | undefined; - if (!embeddingSkipped) { + if (!embeddingSkipped && !options.graphPublication) { const { isHttpMode } = await import('./embeddings/http-client.js'); const httpMode = isHttpMode(); progress( @@ -5375,10 +5815,13 @@ async function runFullAnalysisInner( // final meta, written after the swap; never publish it at a checkpoint. const latestMetaForCount = embeddingCount === undefined ? ((await loadMeta(metaDir)) ?? existingMeta) : undefined; - const persistedEmbeddingCount = resolvePersistedEmbeddingCount( - measuredEmbeddingCount, - stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, - ); + const persistedEmbeddingCount = + options.graphPublication && existingMeta?.vectorGeneration && !options.dropEmbeddings + ? existingMeta.vectorGeneration.chunks + : resolvePersistedEmbeddingCount( + measuredEmbeddingCount, + stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, + ); const { getRuntimeCapabilities } = await import('./platform/capabilities.js'); const runtimeCapabilities = getRuntimeCapabilities(); @@ -5423,9 +5866,17 @@ async function runFullAnalysisInner( // RepoMeta's status unions (tri-review 4669518496 P1/U3) — an unannotated // literal widens the vectorSearch.status ternary to `string` and the // honesty contract silently decays to "whatever interpolates". - const meta: RepoMeta = { + let meta: RepoMeta = { repoPath, storagePath, + ...(options.graphPublication && !options.dropEmbeddings + ? { + graphGeneration: existingMeta?.graphGeneration, + vectorGeneration: existingMeta?.vectorGeneration, + embeddingJob: existingMeta?.embeddingJob, + legacyVectorSource: existingMeta?.legacyVectorSource, + } + : {}), ...(path.basename(lbugPath) !== LBUG_DIRECTORY ? { graphPath: path.basename(lbugPath) } : {}), lastCommit: currentCommit, indexedAt: new Date().toISOString(), @@ -5545,7 +5996,8 @@ async function runFullAnalysisInner( // dirty flag (full and incremental success paths converge here). // Derived digest of the DDL this run created the tables from (#2798). // Git-only: non-git repos never take the incremental path. - schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, + schemaFingerprint: + options.graphPublication || hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), // Resolution runs again on warm parse-cache hits. Replace the receipt, // including complete-empty, so corrected/deleted source clears old gaps. @@ -5607,7 +6059,12 @@ async function runFullAnalysisInner( incrementalInProgress: undefined as RepoMeta['incrementalInProgress'], // Cleared on a clean run; otherwise the marker Phase 4/5 minted above // (see the `kind` doc in repo-manager.ts). - embeddingCheckpoint: pendingEmbeddingCheckpoint, + embeddingCheckpoint: + options.graphPublication && !options.dropEmbeddings + ? resumeEmbeddingCheckpoint + ? existingMeta?.embeddingCheckpoint + : undefined + : pendingEmbeddingCheckpoint, // The effective pdg config this run's DB rows were built under // (#2099 F1). `undefined` on pdg-off runs — this meta is a fresh // literal (no spread of existingMeta), so omission is what CLEARS the @@ -5658,11 +6115,88 @@ async function runFullAnalysisInner( // Stop the manual checkpoint driver before closeLbug so its // in-flight CHECKPOINT cannot race the `safeClose` CHECKPOINT. await walCheckpointDriver.stop(); - if (useAtomicSwap && !wroteChangedSubgraphOnly && !(await runCheckpointWithRetry()).flushed) { + if ( + (useAtomicSwap || options.graphPublication) && + !wroteChangedSubgraphOnly && + !(await runCheckpointWithRetry()).flushed + ) { // Publication requires durability even when periodic/manual checkpoints // are disabled. A best-effort close cannot certify the staged file. throw new Error('Cannot publish staged index: final checkpoint could not be verified.'); } + let preparedGraph: PreparedAnalysisHandoff | undefined; + if (options.graphPublication) { + if (!meta.parseCacheIdentity) + throw new Error('Graph publication requires verified parser provenance'); + if (!existingMeta?.graphGeneration && !options.dropEmbeddings && !meta.legacyVectorSource) { + const { exportLegacyVectors } = await import('./lbug/legacy-vector-export.js'); + const historical = existingMeta?.embeddingCheckpoint; + meta.legacyVectorSource = await exportLegacyVectors({ + metadataDirectory: metaDir, + query: executeQuery, + provenance: { + schemaFingerprint: existingMeta?.schemaFingerprint ?? SCHEMA_FINGERPRINT, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: new Set([ + ...(pendingEmbeddingNodeIds ?? []), + ...inheritedUnsafeNodeIds, + ...restoreFailedNodeIds, + ]), + bufferPoolBytes: options.resources!.embedding.vectorPoolBytes, + threads: options.resources!.embedding.nativeThreads, + }); + } + if (meta.embeddingCheckpoint?.recovery) { + // The complete staged groups now live in the independent legacy source. + // Publish the remaining retry decision without the retired staging + // pointer; the embedding continuation consumes the durable source. + meta.embeddingCheckpoint = { ...meta.embeddingCheckpoint }; + delete meta.embeddingCheckpoint.recovery; + } + const content = createHash('sha256'); + for (const entry of [...newFileHashes].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) + content.update(JSON.stringify(entry)).update('\0'); + const receipt = await prepareAnalysisPublication( + metaDir, + buildPath, + meta, + content.digest('hex'), + useAtomicSwap + ? { waitForProcessExit: options.graphPublication === 'after-exit' } + : { + mode: 'in-place', + attemptId: inPlaceAttempt!, + waitForProcessExit: options.graphPublication === 'after-exit', + }, + ); + if (parseCache) { + try { + await stageAnalysisParseCache(metaDir, receipt.id, parseCache); + } catch (error) { + log(`Warning: could not stage parser cache (${(error as Error).message}); continuing.`); + } + } + if (options.graphPublication === 'after-exit') { + preparedGraph = { + version: 1, + publicationId: receipt.id, + repoPath, + storagePath, + metadataDirectory: metaDir, + repoName: await preparedAnalysisName(repoPath, options), + commit: currentCommit, + }; + } + } // CLI callers (about to process.exit) skip the native close to dodge a // LadybugDB destructor double-free after --pdg writes — closeLbugBeforeExit // CHECKPOINTs for durability then leaves the handles for process exit to @@ -5672,7 +6206,8 @@ async function runFullAnalysisInner( // same-process open file can't be renamed), so it forces a real close — // safe because windowsSwapOk excludes --pdg (the #2264 case). POSIX renames // an open file, so it keeps the skip-native-close there. - const forceRealCloseForSwap = useAtomicSwap && process.platform === 'win32'; + const forceRealCloseForSwap = + useAtomicSwap && process.platform === 'win32' && !options.graphPublication; await (options.skipNativeCloseOnExit && !forceRealCloseForSwap ? closeLbugBeforeExit() : closeLbug()); @@ -5696,7 +6231,7 @@ async function runFullAnalysisInner( () => false, ) : false; - if (useAtomicSwap && builtDbExists) { + if (!options.graphPublication && useAtomicSwap && builtDbExists) { await assertLbugPublicationReady(buildPath); const releaseInitLock = await acquireInitLock(lbugPath); try { @@ -5714,8 +6249,22 @@ async function runFullAnalysisInner( // than latching on the old one. Leaving the dirty flag set across the swap // is a crash-safety improvement: a failed swap leaves the previous index // live and the next run recovers via the full-rebuild path. - await saveMeta(metaDir, meta); - sweepStagingArtifacts(metaDir, log); + if (options.graphPublication === 'inline') { + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + const published = await publishPreparedAnalysis(metaDir, { + saveMetadata: (metadata) => saveMeta(metaDir, metadata), + verify: (file, metadata) => + verifyPublishedAnalysisGraph(file, metadata, { + bufferPoolBytes: options.resources!.parse.graphPoolBytes, + threads: options.resources!.parse.nativeThreads, + }), + }); + if (!published) throw new Error('Prepared graph disappeared before publication'); + meta = published; + } else if (!preparedGraph) { + await saveMeta(metaDir, meta); + } + if (!preparedGraph) sweepStagingArtifacts(metaDir, log); // Registry freshness is published only after the graph and its metadata. // A failed close, swap, or metadata save must leave the previous registry @@ -5729,17 +6278,19 @@ async function runFullAnalysisInner( // (after applying the precedence chain in registerRepo) — reuse it // so AGENTS.md / skill files reference the same name MCP clients // will look up (#979). - const projectName = await registerRepo(repoPath, meta, { - name: options.registryName, - onRename: (previousName, nextName) => - log(`Registry name changed: "${previousName}" -> "${nextName}".`), - allowDuplicateName: options.allowDuplicateName, - // Non-primary branch runs upsert into the entry's branches[]; the - // primary/flat run (placement.branch === undefined) refreshes the - // top-level fields (#2106). - branch: placement.branch, - storagePath, - }); + const projectName = + preparedGraph?.repoName ?? + (await registerRepo(repoPath, meta, { + name: options.registryName, + onRename: (previousName, nextName) => + log(`Registry name changed: "${previousName}" -> "${nextName}".`), + allowDuplicateName: options.allowDuplicateName, + // Non-primary branch runs upsert into the entry's branches[]; the + // primary/flat run (placement.branch === undefined) refreshes the + // top-level fields (#2106). + branch: placement.branch, + storagePath, + })); // ── #2354: the flat workspace slot has adopted this run's branch ────── // Drop a now-shadowed `branches//` sub-index for the same label @@ -5751,7 +6302,7 @@ async function runFullAnalysisInner( // what applyBranchScope trusts. Retried by the next content-changing run // (same-commit fast-path runs skip it: their guard compares the // already-stamped meta label). - if (!placement.branch && branchLabel) { + if (!preparedGraph && !placement.branch && branchLabel) { try { await adoptFlatBranchLabel(repoPath, branchLabel, storagePath); } catch (e) { @@ -5814,11 +6365,12 @@ async function runFullAnalysisInner( // composition no longer matches anything in the current scan are dead // weight; the parse phase populates `usedKeys` as it processes chunks). try { - if (parseCache) { + if (parseCache && !preparedGraph) { await publishAnalysisParseCache(parseCache, { storagePath, metadataDirectory: metaDir, cacheRoot, + publishedGeneration: meta.graphGeneration?.id, sharedStore: writeTarget.sharedStore, collectBranchKeys: collectBranchCacheKeys, log, @@ -5828,9 +6380,13 @@ async function runFullAnalysisInner( log(`Warning: could not save parse cache (${(e as Error).message}); continuing.`); } - progress('done', 100, 'Done'); + if (preparedGraph) + progress('graph-prepared', 98, 'Graph checkpointed; awaiting process release'); + else if (options.graphPublication) + callbacks.onProgress('graph-ready', 98, 'Graph published', { graphPhase: 'ready' }); + else progress('done', 100, 'Done'); - await removeColdParseRebuildDir(coldParseRebuildDir, true); + if (!preparedGraph) await removeColdParseRebuildDir(coldParseRebuildDir, true); return { repoName: projectName, @@ -5839,6 +6395,7 @@ async function runFullAnalysisInner( stats: meta.stats, rebuildReasons: collector.keys(), pipelineResult, + ...(preparedGraph ? { preparedGraph } : {}), ...(graphWriteCollapsed ? { graphWriteCollapsed } : {}), ftsSkipped: !ftsReady, ftsSkipReason: ftsReady ? undefined : ftsSkipReason, @@ -5893,7 +6450,8 @@ async function runFullAnalysisInner( // differs from `lbugPath` exactly when the atomic-swap plan is in effect, // and the live index is never that path. Best-effort by construction — the // rethrow below is the surface, and the lock's sweep remains the backstop. - if (stagedBuild) { + const retainPrepared = options.graphPublication && (await retainsPreparedGraph(metaDir)); + if (stagedBuild && !retainPrepared) { try { const recovery = resolveEmbeddingRecovery( metaDir, @@ -5906,7 +6464,7 @@ async function runFullAnalysisInner( /* swallow — orphan reclamation must never mask the real failure */ } } - await removeColdParseRebuildDir(coldParseRebuildDir, true); + if (!retainPrepared) await removeColdParseRebuildDir(coldParseRebuildDir, true); discardCachedEmbeddingSpill(); // What the caller sees: `err` itself (identity and prototype preserved, callers // tell IndexLockTimeoutError and other domain failures apart with `instanceof`), diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 717873b21..8168a1557 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -525,6 +525,9 @@ export const publishSharedGraph = async ( // Embeddings still owed stay with this checkout, which finishes them; a // commit graph never changes, so a shared copy would stay short for good. !meta.embeddingCheckpoint && + // A migration receipt owns historical vectors in this private slot. Do + // not replace its graph or strip that evidence before migration finishes. + !Object.hasOwn(meta, 'legacyVectorSource') && // A sparse or partial checkout builds a graph missing the files it hides. // Every sparse mode marks those entries skip-worktree, which this rejects // (git-utils.test.ts covers no-cone, cone and sparse-index checkouts). diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index 5a6319ac0..df1f1ebee 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -89,7 +89,7 @@ export interface AnalyzeJob { const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const JOB_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes -/** How long a cancelled worker gets to exit via IPC before a signal is sent. */ +/** How long to retain event-loop references after requesting safe cancellation. */ const CANCEL_GRACE_MS = 15_000; export class JobManager { @@ -209,20 +209,27 @@ export class JobManager { /** Register a child process for a job — enables cancellation and timeout. */ registerChild(jobId: string, child: ChildProcess) { + const previousTimer = this.timeouts.get(jobId); + if (previousTimer) clearTimeout(previousTimer); this.children.set(jobId, child); - // 30-minute timeout - const timer = setTimeout(() => { - const job = this.jobs.get(jobId); - if (job && !this.isTerminal(job.status)) { - this.cancelJob(jobId, 'Analysis timed out (30 minute limit)'); - } - }, JOB_TIMEOUT_MS); + // One deadline covers graph preparation, continuation, and crash retries. + const elapsed = Date.now() - (this.jobs.get(jobId)?.startedAt ?? Date.now()); + const timer = setTimeout( + () => { + const job = this.jobs.get(jobId); + if (job && !this.isTerminal(job.status)) { + this.cancelJob(jobId, 'Analysis timed out (30 minute limit)'); + } + }, + Math.max(0, JOB_TIMEOUT_MS - elapsed), + ); this.timeouts.set(jobId, timer); // Apply a pending cancel BEFORE other `exit` listeners (analyze-launch's // crash-retry) see a still-non-terminal job and fork a replacement worker. const onExit = (): void => { + if (this.children.get(jobId) !== child) return; this.releaseChild(jobId); this.applyPendingCancel(jobId); }; @@ -289,10 +296,8 @@ export class JobManager { * * The worker is asked to stop over IPC first (`{ type: 'cancel' }`), which * lets it reach a JS-visible safe point and checkpoint before exiting — - * the same cross-platform control path `core/auto-sync` uses. A signal is - * sent only as a bounded fallback: on Windows `child.kill('SIGTERM')` is a - * forceful termination (Node ignores the signal name there), so leading - * with it could kill the worker mid LadybugDB write. + * the same cross-platform control path `core/auto-sync` uses. Native work + * must reach a safe point even if that takes longer than the grace period. */ cancelJob(jobId: string, reason?: string): boolean { const job = this.jobs.get(jobId); @@ -329,32 +334,25 @@ export class JobManager { } /** - * Ask a worker to shut down: IPC cancel now, signal after a grace period if - * it has not exited on its own. The grace timer is cleared by the child's - * `exit` handler registered in `registerChild`. + * Request cancellation, then abandon event-loop references after grace. + * Ownership and the job slot remain held until the actual process exit. */ private requestChildShutdown(jobId: string, child: ChildProcess): void { - let ipcSent = false; if (child.connected) { try { - child.send({ type: 'cancel' }); - ipcSent = true; + child.send({ type: 'cancel' }, () => { + /* Disconnect also requests cancellation. */ + }); } catch { - // Channel already closed — fall through to the signal path. + // The child's disconnect handler cancels without interrupting native work. } } - if (!ipcSent) { - child.kill('SIGTERM'); - return; - } if (this.cancelGraceTimers.has(jobId)) return; const grace = setTimeout(() => { this.cancelGraceTimers.delete(jobId); if (child.exitCode === null && child.signalCode === null) { - // IPC already set the worker's cooperative cancel flag; a second - // SIGTERM is a no-op there. SIGKILL is the actual bounded fallback - // (on Windows `kill()` is already TerminateProcess). - child.kill('SIGKILL'); + child.channel?.unref(); + child.unref?.(); } }, CANCEL_GRACE_MS); grace.unref?.(); @@ -362,23 +360,14 @@ export class JobManager { } dispose() { - // IPC first so a worker that checks in can stop at a safe point. - // On Windows `child.kill('SIGTERM')` is TerminateProcess — skip that - // immediate kill and leave the 15s grace timer to SIGKILL. On Unix, - // SIGTERM after IPC is cooperative, so the grace timers can be dropped. - const windows = process.platform === 'win32'; + // Never turn server shutdown into a native-active worker kill. for (const [jobId, child] of this.children) { this.requestChildShutdown(jobId, child); - if (!windows) { - child.kill('SIGTERM'); - } + child.channel?.unref(); + child.unref?.(); } - this.children.clear(); - if (!windows) { - for (const timer of this.cancelGraceTimers.values()) clearTimeout(timer); - this.cancelGraceTimers.clear(); - } - this.pendingCancelReasons.clear(); + for (const timer of this.cancelGraceTimers.values()) clearTimeout(timer); + this.cancelGraceTimers.clear(); for (const controller of this.abortControllers.values()) controller.abort(); this.abortControllers.clear(); diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts index 123c64d72..a79518976 100644 --- a/gitnexus/src/server/analyze-launch.ts +++ b/gitnexus/src/server/analyze-launch.ts @@ -17,7 +17,6 @@ import { fork } from 'child_process'; import { fileURLToPath, pathToFileURL } from 'url'; import { createRequire } from 'node:module'; import { INDEX_METADATA_FILE } from '../storage/repo-manager.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { ANALYZE_FORCE_STORAGE_REQUIREMENTS, ANALYZE_STORAGE_REQUIREMENTS, @@ -25,9 +24,15 @@ import { } from '../storage/storage-resolver.js'; import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js'; import { logger } from '../core/logger.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; +import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; +import type { ContinueMessage, GraphPreparedMessage } from './analyze-worker-protocol.js'; import { loadAnalyzeConfig } from '../cli/analyze-config.js'; import type { RouteHookConfig } from '../core/ingestion/language-provider.js'; -import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; import { isTerminalJobStatus, type JobManager } from './analyze-job.js'; import type { WorkerMessage } from './analyze-worker.js'; import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; @@ -165,7 +170,7 @@ const waitForSettledIndex = async ( } } - const lbugPath = path.resolve(probeRoot, LBUG_DIRECTORY); + const lbugPath = path.resolve(resolveGraphPath(probeRoot)); const lbugRel = path.relative(storageRoot, lbugPath); if (lbugRel.startsWith('..') || path.isAbsolute(lbugRel)) { return false; @@ -173,8 +178,8 @@ const waitForSettledIndex = async ( const lbugStat = statSync(lbugPath); if (lbugStat.mtimeMs < jobStartMs) return false; - return ['lbug.wal', 'lbug.shadow', 'lbug.wal.checkpoint'].every((name) => { - const sidePath = path.resolve(probeRoot, name); + return ['.wal', '.shadow', '.wal.checkpoint'].every((suffix) => { + const sidePath = path.resolve(`${lbugPath}${suffix}`); const sideRel = path.relative(storageRoot, sidePath); if (sideRel.startsWith('..') || path.isAbsolute(sideRel)) { return false; @@ -224,6 +229,12 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // For waitForSettledIndex: files (re)written by this job have mtimes at or // after this instant. Taken before the fork so no worker write predates it. const jobStartMs = Date.now(); + const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); + const resources = resolveHostAnalysisResources({ + mainHeapMb: Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb + 48 : undefined, + includeEmbeddings: await analysisMayEmbed(targetPath, opts), + supervisorBytes: analysisSupervisorReservation(), + }); const analyzeLockKey = await requireStoragePath( targetPath, opts.force ? ANALYZE_FORCE_STORAGE_REQUIREMENTS : ANALYZE_STORAGE_REQUIREMENTS, @@ -271,6 +282,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { 'analyze launch: forwarding route configuration to the worker', ); + const analysisOptions = { + ...opts, + graphPublication: 'auto' as const, + resources, + routeHookConfig, + }; + jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); // ── Worker fork with auto-retry ────────────────────────────── @@ -282,20 +300,12 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] : []; - // Worker heap: 8192MB historical default, but never above what this - // machine/container actually has (#2649 review — a fixed 8192 inside a - // smaller cgroup limit died to the kernel with a misleading remedy). - // GITNEXUS_SERVER_ANALYZE_HEAP_MB overrides as an absolute value. - const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); - const workerHeapMb = - Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb : Math.min(8192, autoHeapCapMb()); - const launchAborted = (jobId: string): boolean => { const current = jobManager.getJob(jobId); return !current || isTerminalJobStatus(current.status) || jobManager.hasPendingCancel(jobId); }; - const forkWorker = () => { + const forkWorker = (continuation?: ContinueMessage) => { if (launchAborted(job.id)) { // Cancelled (or timed out) between lock acquisition and the fork, or // during a crash-retry delay. A pending-cancel job stays non-terminal @@ -306,7 +316,11 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { } const child = fork(workerPath, [], { - execArgv: [...tsxHookArgs, `--max-old-space-size=${workerHeapMb}`], + execArgv: [ + ...tsxHookArgs, + `--max-old-space-size=${resources.mainHeapMb - 48}`, + '--max-semi-space-size=16', + ], stdio: ['ignore', 'pipe', 'pipe', 'ipc'], }); @@ -326,12 +340,15 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // "don't release on cancel error" would leak the lock. let holdLockUntilExit = false; let childExited = false; + let graphVisible = false; + let prepared: GraphPreparedMessage | undefined; + let terminalMessage: Extract | undefined; child.stderr?.on('data', (chunk: Buffer) => { stderrChunks += chunk.toString(); if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); }); - child.on('message', (msg: WorkerMessage) => { + const handleMessage = (msg: WorkerMessage) => { // Ignore any message once the job is terminal — a late worker message (a // SIGTERM-driven `error` after `complete`, or vice versa) must not // re-release the repo lock or flip the reported status. Mirrors the `exit` @@ -339,6 +356,52 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { const current = jobManager.getJob(job.id); if (!current || isTerminalJobStatus(current.status)) return; + if ((msg.type === 'complete' || msg.type === 'error') && !childExited) { + terminalIpcSeen = true; + terminalMessage ??= msg; + if (jobManager.applyPendingCancel(job.id)) holdLockUntilExit = true; + return; + } + if (msg.type === 'graph-prepared') { + try { + if (prepared || continuation) throw new Error('Repeated graph preparation'); + prepared = { ...msg, handoff: projectPreparedAnalysisHandoff(msg.handoff) }; + } catch (error) { + terminalMessage = { type: 'error', message: (error as Error).message }; + terminalIpcSeen = true; + child.send({ type: 'cancel' }); + } + return; + } + if (msg.type === 'graph-ready') { + // Core has closed the writer and verified publication. Refresh readers + // while this job is still active, then admit the independent vector phase. + void closeDbHandle() + .then(() => backend.init()) + .then(() => { + if (childExited) return; + if (launchAborted(job.id)) child.send({ type: 'cancel' }); + else { + graphVisible = true; + child.send({ type: 'graph-visible' }); + } + }) + .catch((error) => { + if (childExited) return; + terminalMessage = { + type: 'error', + message: `Server failed to reload the published graph: ${(error as Error).message}`, + }; + terminalIpcSeen = true; + try { + child.send({ type: 'cancel' }); + } catch { + /* Wait for actual exit. */ + } + }); + return; + } + if (msg.type === 'complete' || msg.type === 'error') terminalIpcSeen = true; if (msg.type === 'progress') { @@ -381,20 +444,21 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // would spin the full 60s and hold the single global analyze slot. // ftsRepairedOnly DOES rewrite `lbug` (initLbug + createSearchFTSIndexes) // so it still waits. - const settle = msg.result.alreadyUpToDate - ? Promise.resolve(true) - : waitForSettledIndex( - // A checkout's first analyze may write a shared-store slot the - // launcher's pre-run lookup could not see yet (#3352); the - // worker reports it. - (msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath) - ? msg.result.storagePath - : null) ?? analyzeLockKey, - jobStartMs, - opts.branch, - msg.result.isPrimaryBranch, - () => launchAborted(job.id), - ); + const settle = + continuation || graphVisible || msg.result.alreadyUpToDate + ? Promise.resolve(true) + : waitForSettledIndex( + // A checkout's first analyze may write a shared-store slot the + // launcher's pre-run lookup could not see yet (#3352); the + // worker reports it. + (msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath) + ? msg.result.storagePath + : null) ?? analyzeLockKey, + jobStartMs, + opts.branch, + msg.result.isPrimaryBranch, + () => launchAborted(job.id), + ); settle .then((settled) => { if (launchAborted(job.id)) { @@ -515,7 +579,8 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { jobManager.updateJob(job.id, { status: 'failed', error: msg.message }); } } - }); + }; + child.on('message', handleMessage); child.on('error', (err) => { // Fake test children have no `pid`. Treat that as pre-spawn so the @@ -539,6 +604,18 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { child.on('exit', (code) => { childExited = true; + if (terminalMessage) { + handleMessage( + code === 0 + ? terminalMessage + : { + type: 'error', + message: `Analyze worker exited unsuccessfully (${code ?? 'signal'})`, + }, + ); + if (launchAborted(job.id)) releaseLockOnce(); + return; + } const j = jobManager.getJob(job.id); if (!j || isTerminalJobStatus(j.status) || jobManager.hasPendingCancel(job.id)) { // (a) complete/error IPC is in flight (`terminalIpcSeen`) — that @@ -560,6 +637,17 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // finished, successful analysis. if (terminalIpcSeen) return; + if (prepared && code === 0 && child.pid) { + forkWorker({ + type: 'continue', + repoPath: targetPath, + options: analysisOptions, + writerPid: child.pid, + prepared, + }); + return; + } + // Worker crashed — attempt retry if under the limit if (j.retryCount < MAX_WORKER_RETRIES) { j.retryCount++; @@ -578,7 +666,7 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { }, }); stderrChunks = ''; - setTimeout(forkWorker, delay); + setTimeout(() => forkWorker(), delay); } else { // Exhausted retries — permanent failure releaseLockOnce(); @@ -593,20 +681,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { jobManager.registerChild(job.id, child); // Send start command to child - child.send({ - type: 'start', - repoPath: targetPath, - options: { - force: !!opts.force, - embeddings: !!opts.embeddings, - dropEmbeddings: !!opts.dropEmbeddings, - ...(opts.springActuatorPath ? { springActuatorPath: opts.springActuatorPath } : {}), - ...(opts.asyncApiSpecPath ? { asyncApiSpecPath: opts.asyncApiSpecPath } : {}), - ...(opts.registryName ? { registryName: opts.registryName } : {}), - ...(opts.branch ? { branch: opts.branch } : {}), - routeHookConfig, + child.send( + continuation ?? { + type: 'start', + repoPath: targetPath, + options: analysisOptions, }, - }); + ); }; try { diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index 477d8910f..825b0a5b3 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -18,7 +18,7 @@ import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; // `analyze-worker.ts` entry module: importing it from the entry made this // module depend on the very module that depends on it (#cycle). Erased at // runtime either way; this way the graph is acyclic too. -import type { WorkerMessage } from './analyze-worker-protocol.js'; +import type { WorkerMessage, ContinueMessage } from './analyze-worker-protocol.js'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; @@ -28,6 +28,9 @@ import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; + continuePreparedAnalysis?: typeof import('../core/run-analyze.js').continuePreparedAnalysis; + onGraphPublished?: () => Promise; + signal?: AbortSignal; assertAnalysisFinalized: typeof import('../storage/repo-manager.js').assertAnalysisFinalized; send: (msg: WorkerMessage) => void; /** @@ -53,31 +56,53 @@ export async function runWorkerAnalysis( options: AnalyzeOptions, deps: WorkerAnalysisDeps, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, + continuation?: ContinueMessage, ): Promise { let terminal: WorkerMessage; try { const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap ? [runnerIdentityAtBootstrap] : []; - const result = await deps.runFullAnalysis( - repoPath, - // This worker force-exits right after reporting, so skip the native close - // (it can double-free in LadybugDB's ClientContext destructor after --pdg - // writes); flushWAL still persists the index, process.exit reclaims handles. - { ...options, skipNativeCloseOnExit: true }, - { - onProgress: (phase, percent, message, details) => - deps.send({ - type: 'progress', - phase, - percent, - message, - ...projectAnalysisPhaseProgress(details), - }), - onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), - }, - ...bootstrapArgs, - ); + const callbacks: import('../core/run-analyze.js').AnalyzeCallbacks = { + signal: deps.signal, + onGraphPublished: deps.onGraphPublished, + onProgress: (phase, percent, message, details) => + deps.send({ + type: 'progress', + phase, + percent, + message, + ...projectAnalysisPhaseProgress(details), + }), + onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), + }; + let result: import('../core/run-analyze.js').AnalyzeResult; + if (continuation) { + if (!deps.continuePreparedAnalysis) throw new Error('Missing analysis continuation'); + const handoff = projectPreparedAnalysisHandoff(continuation.prepared.handoff); + if (handoff.repoPath !== repoPath || continuation.prepared.result.repoPath !== repoPath) + throw new Error('Continuation belongs to a different repository'); + const metadata = await deps.continuePreparedAnalysis( + handoff, + continuation.writerPid, + options, + callbacks, + deps.signal, + ); + result = { + ...continuation.prepared.result, + stats: metadata.stats ?? {}, + rebuildReasons: [], + }; + } else { + // Native allocations belong to this exit-only graph process. + result = await deps.runFullAnalysis( + repoPath, + { ...options, skipNativeCloseOnExit: true }, + callbacks, + ...bootstrapArgs, + ); + } if (result.preparedGraph) { if (!deps.isCancelled) throw new Error('Graph handoff requires cancellation state'); if (result.graphWriteCollapsed) diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 28bd9802c..3e6991ffb 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -35,12 +35,25 @@ export interface StartMessage { options: AnalyzeOptions; } +/** A new process may continue only after the graph writer actually exited. */ +export interface ContinueMessage { + type: 'continue'; + repoPath: string; + options: AnalyzeOptions; + writerPid: number; + prepared: GraphPreparedMessage; +} + +export interface GraphVisibleMessage { + type: 'graph-visible'; +} + /** Parent → child: request safe cancellation at the next JS-visible checkpoint. */ export interface CancelMessage { type: 'cancel'; } -export type ParentMessage = StartMessage | CancelMessage; +export type ParentMessage = StartMessage | ContinueMessage | CancelMessage | GraphVisibleMessage; export interface ProgressMessage extends AnalysisPhaseProgress { type: 'progress'; @@ -63,6 +76,11 @@ export interface GraphPreparedMessage { result: AnalyzeResultIpc; } +/** Wait for the serving parent to refresh readers before making paid requests. */ +export interface GraphReadyMessage { + type: 'graph-ready'; +} + export interface ErrorMessage { type: 'error'; message: string; @@ -84,4 +102,9 @@ export interface ErrorMessage { } /** Child → parent IPC messages. Shared with the parent-side launcher. */ -export type WorkerMessage = ProgressMessage | GraphPreparedMessage | CompleteMessage | ErrorMessage; +export type WorkerMessage = + | ProgressMessage + | GraphPreparedMessage + | GraphReadyMessage + | CompleteMessage + | ErrorMessage; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index e83b853e1..ff33526ae 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -74,10 +74,38 @@ process.on('unhandledRejection', (reason: unknown) => { // returned to JS. SIGTERM is retained only for local process shutdown. let cancellationRequested = false; let started = false; +const cancellation = new AbortController(); +let graphVisible: (() => void) | undefined; + +async function waitForGraphVisibility(): Promise { + cancellation.signal.throwIfAborted(); + await new Promise((resolve, reject) => { + const cleanup = () => { + graphVisible = undefined; + cancellation.signal.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); + reject(cancellation.signal.reason); + }; + graphVisible = () => { + cleanup(); + resolve(); + }; + cancellation.signal.addEventListener('abort', onAbort, { once: true }); + try { + send({ type: 'graph-ready' }); + } catch (error) { + cleanup(); + reject(error); + } + }); +} function requestWorkerCancellation(source: string): void { if (cancellationRequested) return; cancellationRequested = true; - if (claimTerminal()) { + cancellation.abort(new Error(`Analysis cancelled (${source})`)); + if (claimTerminal() && process.connected) { send({ type: 'error', message: `Analysis cancelled (${source})` }); } if (!started) { @@ -102,9 +130,14 @@ function exitAfterCancellation(): void { } process.on('SIGTERM', () => requestWorkerCancellation('worker received SIGTERM')); +process.on('disconnect', () => requestWorkerCancellation('supervisor disconnected')); // Listen for parent commands — guarded against re-entry. process.on('message', async (msg: ParentMessage) => { + if (msg.type === 'graph-visible') { + graphVisible?.(); + return; + } if (msg.type === 'cancel') { requestWorkerCancellation('parent requested cancellation'); return; @@ -141,12 +174,16 @@ process.on('message', async (msg: ParentMessage) => { msg.options, { runFullAnalysis: prepared.loaded.analysisModule.runFullAnalysis, + continuePreparedAnalysis: prepared.loaded.analysisModule.continuePreparedAnalysis, + onGraphPublished: msg.options.graphPublication ? waitForGraphVisibility : undefined, + signal: cancellation.signal, assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized, send, claimTerminal, isCancelled: () => cancellationRequested, }, prepared.runnerIdentity, + msg.type === 'continue' ? msg : undefined, ); } catch (error) { if (claimTerminal()) { diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 1eea07846..f2b47150f 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -9,12 +9,12 @@ */ import { - acquireIndexLock, IndexLockTimeoutError, requireExclusiveIndexLock, sweepStagingArtifacts, type IndexLockHandle, } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; import { resolveGraphPath } from '../storage/shared-store.js'; import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; @@ -31,7 +31,6 @@ import { canonicalizePath, cloneDirBelongsToEntry, loadMeta, - saveMeta, listRegisteredRepos, registryPathEquals, type RegistryEntry, @@ -46,9 +45,7 @@ import { import { executeQuery, executePrepared, - executeWithReusedStatement, streamQuery, - flushWAL, closeLbug, withLbugDb, isReadOnlyDbError, @@ -64,24 +61,13 @@ import { contentRetentionFromMeta, isFullSourceAvailable, } from '../core/content-retention.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { getFtsDisabledReason, type FtsDisabledReason } from '../core/search/fts-policy.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js'; import { fileURLToPath } from 'url'; -import { isTerminalJobStatus, JobManager, type AnalyzeJobPartialOutcome } from './analyze-job.js'; +import { isTerminalJobStatus, JobManager } from './analyze-job.js'; import { mountSSEProgress } from './sse-progress.js'; -import { - resolveEmbedRunOutcome, - withMeasuredEmbeddingCount, - type EmbedRunFinalizeContext, -} from './embed-run-outcome.js'; -import { decideEmbeddingResume, mintInterruptedCheckpoint } from '../core/embedding-checkpoint.js'; -import { - measurePersistedEmbeddingCount, - persistedEmbeddingCountOrUndefined, - type PersistedEmbeddingCount, -} from '../core/embedding-count.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; import { assertString, BadRequestError, createRouteLimiter } from './validation.js'; import { parseGrepQuery, GREP_TIME_BUDGET_MS } from './grep-params.js'; import { runGrepScanInWorker } from './grep-scan.js'; @@ -999,6 +985,8 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const app = express(); app.disable('x-powered-by'); const serverStartedAt = Date.now(); + let shuttingDown = false; + const activeEmbeddingRuns = new Set>(); // Which upstream hops may set X-Forwarded-*. Process-wide: every route's // req.ip, and so the per-IP rate limiter, resolves through this. @@ -1067,6 +1055,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const activeRepoPaths = new Set(); const acquireRepoLock = (repoPath: string): string | null => { + if (shuttingDown) return 'Server is shutting down'; if (activeRepoPaths.has(repoPath)) { return `Another job is already active for this repository`; } @@ -1154,20 +1143,28 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) { if (clientGone) return null; // client disconnected — stop polling const currentJob = jobManager.getJob(job.id); - if (!currentJob || currentJob.status === 'failed') { + if ( + currentJob?.progress.graphPhase === 'ready' || + currentJob?.status === 'complete' || + currentJob?.status === 'failed' + ) { + // A published graph is usable while embeddings are pending or + // failed. Re-resolve its owned registry entry before returning. + await backend.init(); + const freshRepos = await listRegisteredRepos({ + validate: options.validateStorage !== false, + }); + const published = resolveRegisteredRepoEntry(freshRepos, repoName); + if (published) return validate(published); + if (currentJob.status === 'complete' || currentJob.status === 'failed') return null; + } + if (!currentJob) { // The job is over and produced no registry entry. This is a // plain "not found", not "still analyzing" — falling through to // the timed-out sentinel here told callers to keep waiting for // a job that had already failed. return null; } - if (currentJob.status === 'complete') { - await backend.init(); - const freshRepos = await listRegisteredRepos({ - validate: options.validateStorage !== false, - }); - return validate(resolveRegisteredRepoEntry(freshRepos, repoName)); - } await new Promise((r) => setTimeout(r, 1000)); } // Timed out — signal to the caller with a specific message @@ -1521,15 +1518,35 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => async () => { let searchResults: any[]; let ftsAvailable: boolean | undefined; + const metadata = mode === 'bm25' ? null : await loadMeta(entry.storagePath); + const snapshot = + metadata && + (metadata.graphGeneration || metadata.vectorGeneration || metadata.embeddingJob) + ? { metadataDirectory: entry.storagePath, metadata, graphPath: lbugPath } + : undefined; + const semanticReady = + mode !== 'bm25' && + (snapshot !== undefined || + (await import('../core/embeddings/embedder.js')).isEmbedderReady()); + const semanticLane = async (execute: typeof executeQuery, text: string, count = 10) => { + const { semanticSearch } = await import('../core/embeddings/embedding-pipeline.js'); + return semanticSearch( + execute, + text, + count, + undefined, + snapshot, + snapshot + ? async (input) => (await import('../mcp/core/embedder.js')).embedQuery(input) + : undefined, + ); + }; if (mode === 'semantic') { - const { isEmbedderReady } = await import('../core/embeddings/embedder.js'); - if (!isEmbedderReady()) { + if (!semanticReady) { return { searchResults: [] as any[], ftsAvailable: undefined }; } - const { semanticSearch: semSearch } = - await import('../core/embeddings/embedding-pipeline.js'); - searchResults = await semSearch(executeQuery, query, limit); + searchResults = await semanticLane(executeQuery, query, limit); // Normalize semantic results to HybridSearchResult shape searchResults = searchResults.map((r: any, i: number) => ({ ...r, @@ -1547,15 +1564,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => })); } else { // hybrid (default) - const { isEmbedderReady } = await import('../core/embeddings/embedder.js'); - if (isEmbedderReady()) { - const { semanticSearch: semSearch } = - await import('../core/embeddings/embedding-pipeline.js'); + if (semanticReady) { searchResults = await hybridSearch( query, limit, executeQuery, - semSearch, + semanticLane, ftsDisabledReason, ); if (ftsDisabledReason) ftsAvailable = false; @@ -2044,7 +2058,6 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }); } })(); - res.status(202).json({ jobId: job.id, status: job.status }); } catch (err: any) { if (err.message?.includes('already in progress')) { @@ -2155,266 +2168,73 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }, EMBED_TIMEOUT_MS); // Run embedding pipeline asynchronously - (async () => { - // Set inside withLbugDb, read after it closes (#2790). - let partialRunError: string | undefined; - let partialRunDetail: AnalyzeJobPartialOutcome | undefined; - // The in-memory repo lock only serializes this server; a CLI analyze - // in another process guards the slot with the index lock, so hold it - // for the whole embedding write, released in the finally below. + const run = (async () => { + // Both server writers and other CLI processes serialize on this slot. let slotLock: IndexLockHandle | undefined; try { - slotLock = await acquireIndexLock(storagePath, { sweep: false }); + slotLock = await acquireAnalysisSlotLock(storagePath, { sweep: false }); requireExclusiveIndexLock( slotLock, `Cannot acquire the index lock at ${storagePath}; refusing an unlocked embedding run.`, ); - // This writer cannot recover staged generations. Preserve their - // receipts, including malformed ones, before sweeping or writing. - const recoveryCheckpoint = (await loadMeta(storagePath))?.embeddingCheckpoint; - if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) { + embedController.signal.throwIfAborted(); + const metadata = await loadMeta(storagePath); + if ( + metadata?.embeddingCheckpoint && + Object.hasOwn(metadata.embeddingCheckpoint, 'recovery') + ) throw new Error( - 'Cannot generate embeddings: the index checkpoint references staged embeddings. ' + - 'Run `gitnexus analyze` to recover them first.', + 'Cannot generate embeddings: the index checkpoint references staged embeddings. Run `gitnexus analyze` to recover them first.', ); - } + const { readPreparedAnalysis } = await import('../core/analysis-publication.js'); + if (await readPreparedAnalysis(storagePath)) + throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.'); sweepStagingArtifacts(storagePath); - // Writes go to the slot's own graph; a shared-store checkout - // reading an immutable commit graph (#3352) takes a private copy. - if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) { + const { resolveHostAnalysisResources } = + await import('../core/analysis-host-resources.js'); + const { getHeapStatistics } = await import('node:v8'); + const resources = resolveHostAnalysisResources({ + includeParsing: false, + includeEmbeddings: true, + mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + }); + if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) throw new Error('The shared graph this repository reads is gone. Re-run analyze.'); - } - const lbugPath = path.join(storagePath, LBUG_DIRECTORY); - const ftsSession = await loadFtsSession(storagePath); - let embeddingMeta = ftsSession.meta; - // Writable embed still replays a leftover FTS-abort WAL. Refuse - // here — doInitLbug only gates the readOnly path, and analyze - // writers must still be able to park/rebuild. + const lbugPath = resolveGraphPath(storagePath); await assertReadOnlyFtsCrashSafe(lbugPath); - await withLbugDb( - lbugPath, - async () => { - const { runEmbeddingPipeline } = - await import('../core/embeddings/embedding-pipeline.js'); - const { resolveEmbeddingIdentity } = - await import('../core/embeddings/embedding-identity.js'); - const embeddingIdentity = resolveEmbeddingIdentity(); - if (!embeddingMeta) { - throw new Error('Repository metadata is missing; run gitnexus analyze first'); - } - const priorCheckpoint = embeddingMeta.embeddingCheckpoint; - // The SAME decision the CLI's resume gate makes - // (core/embedding-checkpoint.ts). This route used to hard-throw on - // any identity mismatch and ignore `attempts` entirely, so a - // `'partial'` marker written by `gitnexus analyze` and resumed - // here hit exactly the permanent wedge `kind` exists to remove: - // two readers of one record disagreeing about the rule it encodes. - // No `force`/`--drop-embeddings` equivalent exists on this route, - // so the flag options go unset and `'discard'` is unreachable — - // it is folded into the abandon arm rather than given an invented - // flag. `maxAttempts` is left to the shared default. - const resume = priorCheckpoint - ? decideEmbeddingResume(priorCheckpoint, embeddingIdentity) - : undefined; - if (resume?.action === 'abort') throw new Error(resume.error); - if (resume?.action === 'abandon' || resume?.action === 'discard') { - logger.warn({ repo: entry.name }, resume.log); - } - const forceReembedNodeIds: ReadonlySet = - resume?.action === 'resume' ? resume.pendingNodeIds : new Set(); - const saveEmbeddingCheckpoint = async ( - checkpoint: { - nodesProcessed: number; - totalNodes: number; - chunksProcessed: number; - }, - pendingNodeIds: string[], - embeddings?: PersistedEmbeddingCount, - ): Promise => { - // tri-review NEW-2: re-read immediately before writing (mirrors - // the pattern in run-analyze.ts's --repair-fts stamp) instead of - // spreading the stale `embeddingMeta` snapshot captured once at - // job start. This job can run up to EMBED_TIMEOUT_MS (30 min); - // without a fresh read, a concurrent writer's update (e.g. a - // --repair-fts capability stamp) would be silently reverted on - // every checkpoint save for the job's whole lifetime. - const latestMeta = (await loadMeta(storagePath)) ?? embeddingMeta; - // `stats.embeddings` only moves when the caller MEASURED the - // live count (the post-flush `onCheckpoint`). The window-start - // callback measures nothing and passes nothing: restating the - // old count there would re-publish a stale number and clobber - // what a preceding `onCheckpoint` just wrote (same split as - // run-analyze.ts's checkpoint writer). - embeddingMeta = withMeasuredEmbeddingCount( - { - ...latestMeta, - // In flight ⇒ `kind: 'interrupted'` (embedding-checkpoint.ts). - embeddingCheckpoint: mintInterruptedCheckpoint( - embeddingIdentity, - checkpoint, - pendingNodeIds, - ), - }, - embeddings, - ); - await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); - await saveMeta(storagePath, embeddingMeta); - }; - /** - * Count the persisted rows, or report the answer never arrived. - * The TRI-STATE is carried to the fold rather than collapsed here: - * `unknown` is not 0, and only the fold knows what to carry - * forward instead (core/embedding-count.ts). - */ - const countPersistedEmbeddings = async (): Promise => { - const counted = await measurePersistedEmbeddingCount(executeQuery); - if (counted.kind === 'unknown') { - logger.warn( - { reason: counted.reason }, - '[embed] could not count persisted embeddings; leaving stats.embeddings untouched', - ); - } - return counted; - }; - // Fetch existing content hashes for incremental embedding. - // Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling. - const { fetchExistingEmbeddingHashes } = - await import('../core/lbug/lbug-adapter.js'); - const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery); - if (existingEmbeddings && existingEmbeddings.size > 0) { - console.log( - `[embed] ${existingEmbeddings.size} nodes already embedded — incremental run with content-hash comparison`, - ); - } - const pipelineResult = await runEmbeddingPipeline( - executeQuery, - executeWithReusedStatement, - (p) => { - embedJobManager.updateJob(job.id, { - progress: { - // `ready` maps to 'finalizing', NOT 'complete' (#2790). - // The pipeline emits `ready`/100% unconditionally before - // returning — including when it dropped nodes to endpoint - // failures — and the route has not measured the index or - // decided the outcome yet, so 'complete' here would make - // the job record contradict itself (`status: 'analyzing'`, - // `progress.phase: 'complete'`). - phase: - p.phase === 'ready' - ? 'finalizing' - : p.phase === 'error' - ? 'failed' - : p.phase, - percent: p.percent, - message: - p.phase === 'loading-model' - ? 'Loading embedding model...' - : p.phase === 'embedding' - ? `Embedding nodes (${p.percent}%)...` - : p.phase === 'indexing' - ? 'Creating vector index...' - : p.phase === 'ready' - ? 'Finalizing embeddings...' - : `${p.phase} (${p.percent}%)`, - }, - }); - }, - {}, // config: use defaults - undefined, // skipNodeIds - existingEmbeddings, - { - source: { - repoPath: entry.path, - fileHashes: embeddingMeta.fileHashes, - contentRetention: contentRetentionFromMeta(embeddingMeta), - }, - signal: embedController.signal, - forceReembedNodeIds, - onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { - await saveEmbeddingCheckpoint(checkpoint, nodeIds); - }, - onCheckpoint: async (checkpoint) => { - // Count AFTER the flush, so the number describes rows that - // are durable rather than rows still pending in the WAL. - await flushWAL(); - await saveEmbeddingCheckpoint( - checkpoint, - [], - await countPersistedEmbeddings(), - ); - }, - }, - ); - - // Flush WAL so subsequent /api/search requests see the new - // embeddings immediately (#1149). In the CLI path closeLbug() - // handles this during process exit, but the server keeps the - // connection open for other routes — a CHECKPOINT is enough. - await flushWAL(); - // Measure inside withLbugDb, after the flush and while the - // connection is still open — this is the route's only chance to - // stamp `stats.embeddings` (embed-run-outcome.ts). A partial run - // gets the same stamp: an honest count of a partial index is what - // makes it survivable. - const measuredEmbeddings = await countPersistedEmbeddings(); - // Same re-read-before-write reasoning as saveEmbeddingCheckpoint - // above — and the outcome decision reads it too: its - // `embeddingCheckpoint` is the marker this run's own mid-run - // writer saved, which is the only record of the work when the - // count query could not answer. - const finalMeta = (await loadMeta(storagePath)) ?? embeddingMeta; - const finalizeContext: EmbedRunFinalizeContext = { - measuredEmbeddings: persistedEmbeddingCountOrUndefined(measuredEmbeddings), - onDisk: finalMeta, - // The marker the job STARTED from — `finalMeta`'s has since been - // overwritten by the in-flight writer, so only this one carries - // the `'partial'` attempt chain. - resumedFrom: priorCheckpoint, - }; - const outcome = resolveEmbedRunOutcome( - embeddingIdentity, - pipelineResult, - finalizeContext, - ); - partialRunError = outcome.error; - partialRunDetail = outcome.partial; - embeddingMeta = withMeasuredEmbeddingCount( - { ...finalMeta, embeddingCheckpoint: outcome.checkpoint }, - measuredEmbeddings, - ); - await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); - await saveMeta(storagePath, embeddingMeta); + await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); + const { runAnalysisEmbeddingPhase } = + await import('../core/analysis-embedding-phase.js'); + await runAnalysisEmbeddingPhase({ + metadataDirectory: storagePath, + graphPath: lbugPath, + resources, + nodeLimit: 0, + expectedGeneration: metadata?.graphGeneration?.id, + signal: embedController.signal, + callbacks: { + onGraphPublished: async () => { + await closeLbug(); + await backend.init(); + }, + onProgress: (phase, percent, message, details) => { + embedJobManager.updateJob(job.id, { + progress: { ...projectAnalysisPhaseProgress(details), phase, percent, message }, + }); + }, + onLog: (message) => logger.info({ repo: entry.name }, message), }, - skipFtsOption(ftsSession.skipFts), - ); - - // Don't overwrite 'failed' if the job was cancelled while the pipeline was running + }); + await backend.init(); const current = embedJobManager.getJob(job.id); - if (!current || current.status !== 'failed') { - // The ONLY terminal event for the job, on both branches — nothing - // earlier maps to a terminal status, and `updateJob` synthesizes - // exactly one event per terminal status (#2264 P3). The explicit - // `progress` keeps the record self-consistent for a poller reading - // `progress.phase` (#2790). - embedJobManager.updateJob( - job.id, - partialRunError === undefined - ? { - status: 'complete', - progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' }, - } - : { - status: 'failed', - error: partialRunError, - // Lets a client separate "retry these N nodes" from "the - // run produced nothing" without a new status member. - partial: partialRunDetail, - progress: { phase: 'failed', percent: 100, message: partialRunError }, - }, - ); - } + if (current && !isTerminalJobStatus(current.status)) + embedJobManager.updateJob(job.id, { + status: 'complete', + progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' }, + }); } catch (err: any) { const current = embedJobManager.getJob(job.id); + logger.error({ err, repo: entry.name }, 'Embedding generation failed'); if (!current || current.status !== 'failed') { embedJobManager.updateJob(job.id, { status: 'failed', @@ -2427,6 +2247,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => releaseRepoLock(repoLockPath); } })(); + activeEmbeddingRuns.add(run); + // Observe both outcomes without creating an unhandled finally promise. + void run.then( + () => activeEmbeddingRuns.delete(run), + () => activeEmbeddingRuns.delete(run), + ); res.status(202).json({ jobId: job.id, status: 'analyzing' }); } catch (err: any) { @@ -2565,11 +2391,16 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // destination is `sync: false` (buffered); `flushLoggerSync()` before // `process.exit` so records emitted during cleanup reach stderr. const shutdown = async () => { + if (shuttingDown) return; + shuttingDown = true; console.log('\nShutting down...'); updateController.stop(); server.close(); jobManager.dispose(); embedJobManager.dispose(); + // Abort requests are cooperative. Keep the ordinary writer locks and + // native handles alive until each in-process embedding run has settled. + await Promise.allSettled(activeEmbeddingRuns); await cleanupMcp(); await closeLbug(); await backend.disconnect(); @@ -2587,12 +2418,10 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // Worker-thread transports (pino-pretty under TTY) handle their own // flush on process exit in v10. `pino.final` was removed in v10 // because the new transport architecture made it unnecessary. - let shuttingDown = false; process.on('uncaughtException', (err) => { logger.error({ err }, 'GitNexus uncaughtException'); flushLoggerSync(); if (!shuttingDown) { - shuttingDown = true; shutdown().catch(() => {}); } }); diff --git a/gitnexus/src/storage/analysis-slot-lock.ts b/gitnexus/src/storage/analysis-slot-lock.ts new file mode 100644 index 000000000..a02beb5a7 --- /dev/null +++ b/gitnexus/src/storage/analysis-slot-lock.ts @@ -0,0 +1,29 @@ +import path from 'node:path'; +import { + acquireIndexLock, + requireExclusiveIndexLock, + type AcquireOptions, + type IndexLockHandle, +} from './index-lock.js'; +import { storeRootOfCheckoutSlot } from './shared-store.js'; + +/** Admit branch writers under the member gate used by shared-store GC. Once + * admitted, the branch lock protects their files without serializing siblings. + * GC holds the gate while listing and trying branch locks, so a new branch + * cannot start writing between that listing and deletion of its member. */ +export async function acquireAnalysisSlotLock( + directory: string, + options: AcquireOptions = {}, +): Promise { + const branches = path.dirname(path.resolve(directory)); + const member = path.dirname(branches); + if (path.basename(branches) !== 'branches' || !storeRootOfCheckoutSlot(member)) + return acquireIndexLock(directory, options); + const gate = await acquireIndexLock(member, { ...options, sweep: false }); + try { + requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${member}.`); + return await acquireIndexLock(directory, options); + } finally { + gate.release(); + } +} diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts index b6353473c..9475d3a28 100644 --- a/gitnexus/src/storage/index-capabilities.ts +++ b/gitnexus/src/storage/index-capabilities.ts @@ -32,6 +32,8 @@ export async function readIndexCapabilities( coverage: selection.coverage ?? 'current', sourceGraphGeneration: selection.generation.graphGeneration, }; + } else if (selection.kind === 'legacy' && metadata?.capabilities?.vectorSearch) { + vectorSearch = { ...metadata.capabilities.vectorSearch, coverage: 'preserved' }; } else { vectorSearch = { provider: 'ladybugdb-vector', diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index 413e7c8b0..4e240d8fa 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -25,12 +25,7 @@ import { storeRootOfCheckoutSlot, type SharedStoreLayout, } from './shared-store.js'; -import { - GITNEXUS_DIR, - INDEX_METADATA_FILE, - LBUG_DIRECTORY, - LEGACY_METADATA_FILE, -} from './storage-constants.js'; +import { GITNEXUS_DIR, INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './storage-constants.js'; type StoreRoot = Pick; @@ -140,7 +135,9 @@ async function hasRetainedAnalysis(slot: string): Promise { const metadata = await loadMetaStrict(directory); if ( metadata && - (Object.hasOwn(metadata, 'embeddingJob') || Object.hasOwn(metadata, 'embeddingCheckpoint')) + (Object.hasOwn(metadata, 'embeddingJob') || + Object.hasOwn(metadata, 'embeddingCheckpoint') || + Object.hasOwn(metadata, 'legacyVectorSource')) ) return true; } @@ -217,8 +214,25 @@ export const reclaimSharedStoreLocked = async ( orphans.delete(slot); continue; } + const branchLocks: IndexLockHandle[] = []; try { - if (lock.lockFree || !(await orphanMembers([slot])).has(slot)) { + let busy = lock.lockFree; + if (!busy) { + for (const branch of (await memberMetadataDirectories(slot)).slice(1)) { + try { + const branchLock = await acquireIndexLock(branch, { timeoutMs: 1, sweep: false }); + branchLocks.push(branchLock); + if (branchLock.lockFree) { + busy = true; + break; + } + } catch { + busy = true; + break; + } + } + } + if (busy || !(await orphanMembers([slot])).has(slot)) { orphans.delete(slot); continue; } @@ -236,6 +250,7 @@ export const reclaimSharedStoreLocked = async ( } result.droppedMembers.push(slot); } finally { + for (const branchLock of branchLocks) branchLock.release(); lock.release(); } } @@ -337,9 +352,7 @@ export const describeSharedGraph = ( graphPath: string, storagePath: string, ): 'shared' | 'private' => - path.resolve(graphPath) === path.join(path.resolve(storagePath), LBUG_DIRECTORY) - ? 'private' - : 'shared'; + path.dirname(path.resolve(graphPath)) === path.resolve(storagePath) ? 'private' : 'shared'; /** Files a shared checkout keeps in `/.gitnexus`; everything else there is legacy. */ const POINTER_DIR_KEEP = new Set([SHARED_STORE_POINTER, '.gitignore', 'run.cjs']); diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index 6319cd6f5..0036611fb 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -73,6 +73,17 @@ export function selectVectorGeneration( !metadataMatchesGraph(metadataDirectory, metadata, graphPath) ) return { kind: 'unavailable', reason: 'graph-generation-unverified' }; + if ( + !vector && + !metadata.embeddingJob && + typeof metadata.legacyVectorSource === 'string' && + DIGEST.test(metadata.legacyVectorSource) && + nonnegative(metadata.stats?.embeddings) && + metadata.stats!.embeddings! > 0 && + nonnegative(metadata.embeddingDims) && + metadata.embeddingDims > 0 + ) + return { kind: 'legacy' }; if (!vector) return { kind: 'unavailable', diff --git a/gitnexus/test/helpers/analysis-http-server.mjs b/gitnexus/test/helpers/analysis-http-server.mjs new file mode 100644 index 000000000..4f833e8af --- /dev/null +++ b/gitnexus/test/helpers/analysis-http-server.mjs @@ -0,0 +1,11 @@ +import http from 'node:http'; +const listen = http.Server.prototype.listen; +http.Server.prototype.listen = function (...args) { + this.once('listening', () => process.send?.({ port: this.address().port })); + return listen.apply(this, args); +}; +const { createServer } = await import('../../dist/server/api.js'); +await createServer(0); +process.on('message', (message) => { + if (message === 'shutdown') process.emit('SIGTERM'); +}); diff --git a/gitnexus/test/helpers/embedding-restart-runner.mjs b/gitnexus/test/helpers/embedding-restart-runner.mjs new file mode 100644 index 000000000..20f0181f3 --- /dev/null +++ b/gitnexus/test/helpers/embedding-restart-runner.mjs @@ -0,0 +1,110 @@ +// Exit at JS-visible boundaries after native promises settle. Never kill a +// worker or interrupt a native call; omit cleanup to exercise process recovery. +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { runPublishedEmbeddingJob } from '../../dist/core/embeddings/embedding-job.js'; +import { VectorStore } from '../../dist/core/lbug/vector-store.js'; +import { EmbeddingJobJournal } from '../../dist/storage/embedding-job-journal.js'; +import { loadMeta, saveMeta } from '../../dist/storage/repo-manager.js'; + +const [root, boundary] = process.argv.slice(2); +const crash = () => process.exit(73); +const secondUnit = (rows) => rows.some((row) => row.nodeId === 'Function:a.ts:n00064'); +const begin = EmbeddingJobJournal.prototype.begin; +EmbeddingJobJournal.prototype.begin = async function (intent) { + if (intent.start.unit === 1 && boundary === 'intent-before') crash(); + await begin.call(this, intent); + if (intent.start.unit === 1 && boundary === 'intent-after') crash(); +}; +for (const method of ['appendBatch', 'checkpointUnit']) { + const original = VectorStore.prototype[method]; + VectorStore.prototype[method] = async function (rows) { + const selected = secondUnit(rows); + if (selected && boundary === `${method}-before`) crash(); + const result = await original.call(this, rows); + if (selected && boundary === `${method}-after`) crash(); + return result; + }; +} +const acknowledge = EmbeddingJobJournal.prototype.acknowledge; +EmbeddingJobJournal.prototype.acknowledge = async function (result) { + const selected = this.snapshot().active?.start.unit === 1; + if (selected && boundary === 'cursor-before') crash(); + await acknowledge.call(this, result); + if (selected && boundary === 'cursor-after') crash(); +}; +const seal = VectorStore.prototype.seal; +VectorStore.prototype.seal = async function (...args) { + const result = await seal.apply(this, args); + if (boundary === 'seal-after') crash(); + return result; +}; + +const nodes = Array.from({ length: 130 }, (_, index) => { + const name = `n${String(index).padStart(5, '0')}`; + const body = + index === 64 + ? Array.from({ length: 40 }, (_, line) => ` const value${line} = "漢😀";`).join('\n') + : ' return 1;'; + const content = `export function ${name}() {\n${body}\n}`; + return { + id: `Function:a.ts:${name}`, + name, + label: 'Function', + filePath: 'a.ts', + content, + startLine: 1, + endLine: content.split('\n').length, + }; +}); +const byId = new Map(nodes.map((node) => [node.id, node])); +const result = await runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, `lbug.generation.${'a'.repeat(32)}`), + resources: { + graphPoolBytes: 16 * 1024 ** 2, + vectorPoolBytes: 64 * 1024 ** 2, + modelBytes: 0, + modelHeapMb: 0, + queueBytes: 16 * 1024 ** 2, + nativeThreads: 1, + reservedBytes: 96 * 1024 ** 2, + }, + config: { threads: 1, subBatchSize: 8, maxInputChars: 256 }, + metadata: { + loadMetadata: () => loadMeta(root), + saveMetadata: async (metadata) => { + await saveMeta(root, metadata); + if (metadata.vectorGeneration && boundary === 'pointer-after') crash(); + }, + }, + }, + { + identity: () => ({ provider: 'local', model: 'restart-fixture', dimensions: 4 }), + initializeProvider: async () => {}, + openSource: async () => ({ + async *scan(label) { + if (label !== 'Function') return; + for (const node of nodes) + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }, + fetch: async (_label, ids) => ids.map((id) => byId.get(id)), + close: async () => {}, + }), + provider: async (texts) => { + const inputs = texts.map((text) => { + // Interior slices of the oversized node need not repeat its name. + const node = text.match(/n\d{5}/)?.[0] ?? null; + return { node, hash: createHash('sha256').update(text).digest('hex') }; + }); + await fs.appendFile( + path.join(root, `${boundary}.inputs.jsonl`), + inputs.map((input) => JSON.stringify(input) + '\n').join(''), + ); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }, + }, +); +await fs.writeFile(path.join(root, `${boundary}.result.json`), JSON.stringify(result)); diff --git a/gitnexus/test/helpers/prepared-analysis-runner.mjs b/gitnexus/test/helpers/prepared-analysis-runner.mjs new file mode 100644 index 000000000..7bfc12151 --- /dev/null +++ b/gitnexus/test/helpers/prepared-analysis-runner.mjs @@ -0,0 +1,48 @@ +// Real exit-only writer: the test controls exit only after all graph work settles. +import { runFullAnalysis } from '../../dist/core/run-analyze.js'; +import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; + +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, +}); +const recovering = process.argv[3] === 'recover'; +const phases = []; +const result = await runFullAnalysis( + process.argv[2], + { + graphPublication: recovering ? 'inline' : 'after-exit', + skipNativeCloseOnExit: !recovering, + resources, + skipFts: true, + skipSkills: true, + skipAgentsMd: true, + }, + { + onProgress(phase) { + phases.push(phase); + }, + }, +); +if (recovering) { + await new Promise((resolve, reject) => + process.send( + { alreadyUpToDate: result.alreadyUpToDate, reasons: result.rebuildReasons, phases }, + (error) => (error ? reject(error) : resolve()), + ), + ); + process.exit(0); +} +if (!result.preparedGraph) throw new Error('Expected a prepared graph'); +await new Promise((resolve, reject) => + process.send({ handoff: result.preparedGraph, stats: result.stats }, (error) => + error ? reject(error) : resolve(), + ), +); +process.once('message', (message) => { + if (message === 'exit') process.exit(0); +}); diff --git a/gitnexus/test/integration/analysis-embedding-flow.test.ts b/gitnexus/test/integration/analysis-embedding-flow.test.ts new file mode 100644 index 000000000..333cf8b3c --- /dev/null +++ b/gitnexus/test/integration/analysis-embedding-flow.test.ts @@ -0,0 +1,176 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer, type Server } from 'node:http'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../dist/core/run-analyze.js'; +import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; +import { loadMeta } from '../../src/storage/repo-manager.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; + +let root: string | undefined; +let server: Server | undefined; +afterEach(async () => { + if (server) await new Promise((resolve) => server!.close(() => resolve())); + server = undefined; + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + if (root) await fs.rm(root, { recursive: true, force: true }); + root = undefined; +}); + +it('publishes before HTTP work and resumes without parsing or repeating durable requests', async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-flow-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + const content = Array.from( + { length: 130 }, + (_, index) => `export function alpha${String(index).padStart(3, '0')}() { return ${index}; }`, + ).join('\n'); + await fs.writeFile(path.join(repo, 'a.ts'), content); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + const accepted: string[] = []; + let fail = true; + let requests = 0; + let graphWasReady = false; + let graphWasVisible = false; + let dimensions = 4; + const endpointErrors: unknown[] = []; + server = createServer(async (request, response) => { + try { + requests++; + expect(graphWasReady).toBe(true); + expect(graphWasVisible).toBe(true); + const published = await loadMeta(slot); + expect(published?.graphGeneration).toBeDefined(); + expect(published?.incrementalInProgress).toBeUndefined(); + expect(published?.embeddingJob).toBeDefined(); + let body = ''; + for await (const chunk of request) body += chunk; + const { input } = JSON.parse(body) as { input: string[] }; + if (fail && accepted.length >= 64) { + response.writeHead(503).end('temporarily unavailable'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: input.map((_text, index) => ({ + index, + embedding: Array.from({ length: dimensions }, (_, dimension) => + dimension === 0 ? 1 : 0, + ), + })), + }), + ); + } catch (error) { + endpointErrors.push(error); + response.writeHead(500).end('fixture failure'); + } + }); + await new Promise((resolve) => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing HTTP fixture port'); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', `http://127.0.0.1:${address.port}/v1`); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'durable-fixture'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '1'); + vi.stubEnv('GITNEXUS_EMBEDDING_SUB_BATCH_SIZE', '8'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, + }); + const options = { + graphPublication: 'inline' as const, + resources, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, + embeddings: true, + embeddingsNodeLimit: 0, + }; + const phases: string[] = []; + const callbacks = { + async onGraphPublished() { + // A serving frontend can finish its asynchronous reader refresh first. + await new Promise((resolve) => setTimeout(resolve, 5)); + graphWasVisible = true; + }, + onProgress(phase: string) { + phases.push(phase); + if (phase === 'graph-ready') graphWasReady = true; + }, + }; + await expect(runFullAnalysis(repo, options, callbacks)).rejects.toThrow(/unfinished|incomplete/); + expect(endpointErrors).toEqual([]); + expect(accepted).toHaveLength(64); + const interrupted = await loadMeta(slot); + expect(interrupted?.graphGeneration).toBeDefined(); + expect(interrupted?.embeddingJob).toBeDefined(); + const source = await EmbeddingGraphSource.open( + path.join(slot, interrupted!.graphPath!), + resources.embedding.graphPoolBytes, + 1, + ); + try { + const functions = []; + for await (const key of source.scan('Function')) functions.push(key.id); + expect(functions).toHaveLength(130); + } finally { + await source.close(); + } + + fail = false; + // A long-lived caller honors the existing provider cooldown before retrying. + const now = Date.now; + vi.spyOn(Date, 'now').mockImplementation(() => now() + 31_000); + phases.length = 0; + // Plain analyze resumes the pending job, without requiring --embeddings. + await runFullAnalysis(repo, { ...options, embeddings: false }, callbacks); + expect(endpointErrors).toEqual([]); + expect(phases).not.toContain('extracting'); + expect(new Set(accepted).size).toBe(accepted.length); + expect(accepted).toHaveLength(130); + const completed = await loadMeta(slot); + expect(completed?.graphGeneration).toEqual(interrupted?.graphGeneration); + expect(completed?.embeddingJob).toBeUndefined(); + expect(completed?.vectorGeneration?.chunks).toBe(130); + const count = requests; + phases.length = 0; + const cached = await runFullAnalysis(repo, options, callbacks); + expect(cached.alreadyUpToDate).toBe(true); + expect(phases).not.toContain('extracting'); + expect(await loadMeta(slot)).toEqual(completed); + expect(requests).toBe(count); + + // A vector-space change regenerates vectors without parsing or republishing + // the graph. Its legacy CodeEmbedding column is no longer the vector store. + dimensions = 8; + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', String(dimensions)); + phases.length = 0; + await runFullAnalysis(repo, options, callbacks); + expect(phases).not.toContain('extracting'); + const resized = await loadMeta(slot); + expect(resized?.graphGeneration).toEqual(completed?.graphGeneration); + expect(resized?.vectorGeneration?.dimensions).toBe(dimensions); + expect(resized?.vectorGeneration?.identity).not.toBe(completed?.vectorGeneration?.identity); + expect(accepted).toHaveLength(260); + + await fs.writeFile(path.join(repo, 'a.ts'), content.replace('return 0;', 'return 999;')); + await runFullAnalysis(repo, options, callbacks); + expect(accepted).toHaveLength(261); + expect(accepted.at(-1)).toContain('return 999;'); + expect((await loadMeta(slot))?.graphGeneration?.id).not.toBe(completed?.graphGeneration?.id); +}, 120_000); diff --git a/gitnexus/test/integration/analysis-publication-flow.test.ts b/gitnexus/test/integration/analysis-publication-flow.test.ts new file mode 100644 index 000000000..5d22c1343 --- /dev/null +++ b/gitnexus/test/integration/analysis-publication-flow.test.ts @@ -0,0 +1,496 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fork, execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { continuePreparedAnalysis, runFullAnalysis } from '../../dist/core/run-analyze.js'; +import { verifyPublishedAnalysisGraph } from '../../dist/core/analysis-graph-verification.js'; +import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { + readPreparedAnalysis, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { + waitForAnalysisProcessExit, + type PreparedAnalysisHandoff, +} from '../../src/core/analysis-phase-supervisor.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import lbug from '@ladybugdb/core'; +import { + openLbugConnection, + closeLbugConnection, + estimateBufferPool, +} from '../../src/core/lbug/lbug-config.js'; +import { readLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; + +let root: string, repo: string, slot: string; +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, +}); +const options = { + graphPublication: 'inline' as const, + resources, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, +}; +const callbacks = { onProgress() {} }; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publication-flow-'))); + repo = path.join(root, 'repo'); + slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 1; }\n'); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); +}); +async function meta(): Promise { + const result = await loadMeta(slot); + if (!result) throw new Error('Missing fixture metadata'); + return result; +} +async function contents(metadata: RepoMeta) { + const source = await EmbeddingGraphSource.open( + path.join(slot, metadata.graphPath!), + 256 * 1024 ** 2, + 1, + ); + try { + const ids = []; + for await (const key of source.scan('Function')) ids.push(key.id); + return (await source.fetch('Function', ids))[0].content; + } finally { + await source.close(); + } +} + +it('keeps automatic plain graph analysis on its existing publication path', async () => { + const beforeWork = vi.fn(() => { + throw new Error('Unexpected process handoff'); + }); + await runFullAnalysis( + repo, + { ...options, graphPublication: 'auto' }, + { + ...callbacks, + onGraphWork: beforeWork, + }, + ); + expect(beforeWork).not.toHaveBeenCalled(); + expect((await meta()).graphGeneration).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'lbug'))).resolves.toBeDefined(); +}, 120_000); + +it.each([128, 1024])( + 'keeps a small graph cache within a %i MiB ceiling', + async (poolMb) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', undefined); + const Database = lbug.Database; + const opened: Array<{ file: string; pool: number }> = []; + vi.spyOn(lbug, 'Database').mockImplementation(function (...args) { + opened.push({ file: args[0], pool: args[1]! }); + return Reflect.construct(Database, args); + }); + const roomy = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 2, + totalMb: 4096, + workers: 1, + graphPoolBytes: poolMb * 1024 ** 2, + modelMemoryMb: 0, + }); + await runFullAnalysis( + repo, + { ...options, graphPublication: 'auto', resources: roomy }, + callbacks, + ); + const writers = opened.filter(({ file }) => file.startsWith(path.join(slot, 'lbug'))); + expect(writers.length).toBeGreaterThan(0); + const expected = Math.min(estimateBufferPool(100), poolMb * 1024 ** 2); + expect(writers.every(({ pool }) => pool === expected)).toBe(true); + }, + 120_000, +); + +it('requests an exit supervisor before parsing or opening a graph writer', async () => { + const phases: string[] = []; + await expect( + runFullAnalysis( + repo, + { + ...options, + graphPublication: 'auto', + embeddings: true, + skipNativeCloseOnExit: true, + }, + { + onProgress: (phase) => phases.push(phase), + onGraphWork: () => { + throw new Error('Supervisor requested'); + }, + }, + ), + ).rejects.toThrow('Supervisor requested'); + expect(phases).not.toContain('extracting'); + expect(await loadMeta(slot)).toMatchObject({ repoPath: repo, lastCommit: '' }); + expect((await loadMeta(slot))?.graphGeneration).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); +}, 120_000); + +it.each([true, false])( + 'publishes a verified native graph before reporting graph-ready (skipFts=%s)', + async (skipFts) => { + const events: string[] = []; + await runFullAnalysis( + repo, + { ...options, skipFts }, + { + onProgress(phase, _percent, _message, details) { + if (phase === 'graph-ready') { + expect(details?.graphPhase).toBe('ready'); + events.push(phase); + } + if (phase === 'done' && _percent === 100) events.push('terminal'); + }, + }, + ); + const published = await meta(); + if (!skipFts) expect(published.capabilities?.fts.status).toBe('available'); + expect(events).toEqual(['graph-ready']); + expect(published.graphGeneration?.coveredContent).toMatch(/^[a-f0-9]{64}$/); + expect(published.graphPath).toMatch(/^lbug\.generation\.[a-f0-9]{32}$/); + expect(await contents(published)).toContain('return 1'); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect( + verifyPublishedAnalysisGraph( + path.join(slot, published.graphPath!), + { + ...published, + stats: { ...published.stats, nodes: (published.stats?.nodes ?? 0) + 1 }, + }, + { bufferPoolBytes: 256 * 1024 ** 2, threads: 1 }, + ), + ).rejects.toThrow('counts differ'); + }, + 120_000, +); + +it('advances the logical generation on a tiny edit without copying its private graph', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + const graph = path.join(slot, first.graphPath!); + const inode = (await fs.stat(graph)).ino; + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 2; }\n'); + const copy = vi.spyOn(fs, 'copyFile'); + await runFullAnalysis(repo, options, callbacks); + const second = await meta(); + expect(second.graphPath).toBe(first.graphPath); + expect((await fs.stat(graph)).ino).toBe(inode); + expect(second.graphGeneration?.id).not.toBe(first.graphGeneration?.id); + expect(second.graphGeneration?.coveredContent).not.toBe(first.graphGeneration?.coveredContent); + expect(second.incrementalInProgress).toBeUndefined(); + expect(await contents(second)).toContain('return 2'); + expect( + copy.mock.calls.filter((args) => + args.some((file) => path.basename(String(file)).startsWith('lbug')), + ), + ).toHaveLength(0); + const unchanged = await runFullAnalysis(repo, options, callbacks); + expect(unchanged.alreadyUpToDate).toBe(true); + expect((await meta()).graphGeneration?.id).toBe(second.graphGeneration?.id); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 5; }\n'); + expect((await runFullAnalysis(repo, options, callbacks)).alreadyUpToDate).not.toBe(true); + expect(await contents(await meta())).toContain('return 5'); + await fs.writeFile(path.join(repo, 'b.ts'), 'export function beta() { return 6; }\n'); + expect((await runFullAnalysis(repo, options, callbacks)).alreadyUpToDate).not.toBe(true); + expect((await meta()).fileHashes?.['b.ts']).toBeDefined(); +}, 120_000); + +it('preserves completed vector coverage across a new graph generation', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + const vector = { + version: 1 as const, + graphGeneration: first.graphGeneration!.id, + identity: 'a'.repeat(64), + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact' as const, + }; + await saveMeta(slot, { ...first, vectorGeneration: vector }); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 3; }\n'); + await runFullAnalysis(repo, options, callbacks); + const second = await meta(); + expect(second.vectorGeneration).toEqual({ + ...vector, + preservedForGraph: second.graphGeneration?.id, + }); + expect(second.stats?.embeddings).toBe(1); +}, 120_000); + +it('preserves an old-width legacy table before writable schema initialization changes it', async () => { + await runFullAnalysis(repo, { ...options, graphPublication: 'auto' }, callbacks); + const graph = path.join(slot, 'lbug'); + const handle = await openLbugConnection(lbug, graph, { + bufferPoolBytes: 256 * 1024 ** 2, + threads: 1, + }); + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) cursor.close(); + }; + try { + await query('DROP TABLE CodeEmbedding'); + await query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ); + await query( + "CREATE (:CodeEmbedding {id:'old:0',nodeId:'old',chunkIndex:0,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'historical'})", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graph }); + } + await saveMeta(slot, { + ...(await meta()), + embeddingDims: 4, + stats: { ...(await meta()).stats, embeddings: 1 }, + }); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 7; }\n'); + await runFullAnalysis(repo, options, callbacks); + const published = await meta(); + expect(published.legacyVectorSource).toMatch(/^[a-f0-9]{64}$/); + const retained = await readLegacyEmbeddingSource(slot, published.legacyVectorSource!); + expect(retained.recordedIdentity).toBeUndefined(); + const old = await openLbugConnection(lbug, retained.path, { + readOnly: true, + bufferPoolBytes: 32 * 1024 ** 2, + threads: 1, + }); + try { + const result = await old.conn.query('MATCH (e:CodeEmbedding) RETURN e.embedding AS vector'); + const cursors = Array.isArray(result) ? result : [result]; + try { + expect(await cursors[0].getAll()).toEqual([{ vector: [1, 0, 0, 0] }]); + } finally { + for (const cursor of cursors) cursor.close(); + } + } finally { + await closeLbugConnection(old, { strict: true, databasePath: retained.path }); + } + expect(await contents(published)).toContain('return 7'); +}, 120_000); + +it('retains an unpublished replacement and its parse cache when publication fails', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 4; }\n'); + const rename = fs.rename; + const failPublication = vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + if ( + String(source).includes('.staging.') && + path.basename(String(destination)).startsWith('lbug.generation.') + ) + throw new Error('publication interrupted'); + return rename(source, destination); + }); + await expect(runFullAnalysis(repo, { ...options, force: true }, callbacks)).rejects.toThrow( + 'publication interrupted', + ); + failPublication.mockRestore(); + expect((await meta()).graphGeneration?.id).toBe(first.graphGeneration?.id); + expect(await contents(await meta())).toContain('return 1'); + const pending = await readPreparedAnalysis(slot); + expect(pending).toBeDefined(); + await expect(fs.stat(path.join(slot, pending!.stagingFile))).resolves.toBeDefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${pending!.id}.json`)), + ).resolves.toBeDefined(); + expect((await fs.readdir(path.join(slot, 'parse-rebuild'))).length).toBeGreaterThan(0); + const recoveryLogs: string[] = []; + const result = await runFullAnalysis(repo, options, { + onLog: (message) => { + recoveryLogs.push(message); + }, + onProgress(phase) { + if (phase === 'extracting') + throw new Error(`Recovery unexpectedly reparsed: ${recoveryLogs.join('\n')}`); + }, + }); + expect(result.alreadyUpToDate).toBe(true); + expect((await meta()).graphGeneration?.id).toBe(pending!.id); + expect(await contents(await meta())).toContain('return 4'); +}, 120_000); + +it.each(['prepared', 'graph-published'] as const)( + 'recovers the %s graph and parse stage after actual writer exit without parsing', + async (state) => { + const child = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + execArgv: [], + env: process.env, + }, + ); + const exit = waitForAnalysisProcessExit(child); + void exit.catch(() => {}); + let output = ''; + child.stderr?.on('data', (chunk) => { + output = (output + String(chunk)).slice(-8192); + }); + const prepared = new Promise<{ handoff: { publicationId: string } }>((resolve, reject) => { + child.once('message', (message) => + resolve(message as { handoff: { publicationId: string } }), + ); + child.once('error', reject); + child.once('exit', (code) => { + if (code !== 0) reject(new Error(`Graph child exited ${code}: ${output}`)); + }); + }); + try { + const message = await prepared; + expect((await readPreparedAnalysis(slot))?.id).toBe(message.handoff.publicationId); + expect((await meta()).graphGeneration).toBeUndefined(); + expect((await fs.readdir(path.join(slot, 'parse-rebuild'))).length).toBeGreaterThan(0); + await expect(runFullAnalysis(repo, options, callbacks)).rejects.toThrow('has not exited'); + child.send('exit'); + await exit; + if (state === 'graph-published') { + // Simulate exit between durable graph publication and cache publication. + await publishPreparedAnalysis(slot, { + saveMetadata: (metadata) => saveMeta(slot, metadata), + verify: (file, metadata) => + verifyPublishedAnalysisGraph(file, metadata, { + bufferPoolBytes: resources.parse.graphPoolBytes, + threads: resources.parse.nativeThreads, + }), + }); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${message.handoff.publicationId}.json`)), + ).resolves.toBeDefined(); + } + // Use the same distribution runtime for both phases: mixing the Vitest and + // plain Node parser loaders would test an analyzer change as well as recovery. + const continuation = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo, 'recover'], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + execArgv: [], + env: process.env, + }, + ); + const continuationExit = waitForAnalysisProcessExit(continuation); + const result = await new Promise<{ + alreadyUpToDate: boolean; + reasons: string[]; + phases: string[]; + }>((resolve, reject) => { + continuation.once('message', (message) => + resolve(message as { alreadyUpToDate: boolean; reasons: string[]; phases: string[] }), + ); + continuation.once('error', reject); + void continuationExit.catch(reject); + }); + await continuationExit; + expect({ alreadyUpToDate: result.alreadyUpToDate, reasons: result.reasons }).toEqual({ + alreadyUpToDate: true, + reasons: [], + }); + expect(result.phases).not.toContain('extracting'); + expect((await meta()).graphGeneration?.id).toBe(message.handoff.publicationId); + expect(await contents(await meta())).toContain('return 1'); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + const keys = (await meta()).cacheKeys!; + expect(keys.length).toBeGreaterThan(0); + for (const key of keys) + await expect(fs.stat(path.join(slot, 'parse-cache', `${key}.v8`))).resolves.toBeDefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${message.handoff.publicationId}.json`)), + ).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + if (child.connected) child.send('exit'); + await exit; + } + }, + 120_000, +); + +it('continues only the exited writer and unchanged target, without repeating forced parsing', async () => { + const child = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'], execArgv: [], env: process.env }, + ); + const exit = waitForAnalysisProcessExit(child); + void exit.catch(() => {}); + child.stderr?.resume(); + child.stdout?.resume(); + try { + const { handoff } = await new Promise<{ handoff: PreparedAnalysisHandoff }>( + (resolve, reject) => { + child.once('message', (message) => + resolve(message as { handoff: PreparedAnalysisHandoff }), + ); + child.once('error', reject); + void exit.then(() => reject(new Error('Writer exited without preparing')), reject); + }, + ); + if (!child.pid) throw new Error('Missing writer PID'); + await expect(continuePreparedAnalysis(handoff, child.pid, options, callbacks)).rejects.toThrow( + 'has not exited', + ); + child.send('exit'); + await exit; + await expect( + continuePreparedAnalysis( + handoff, + child.pid, + { ...options, registryName: 'changed' }, + callbacks, + ), + ).rejects.toThrow('registered name changed'); + expect((await readPreparedAnalysis(slot))?.id).toBe(handoff.publicationId); + const phases: string[] = []; + const published = await continuePreparedAnalysis( + handoff, + child.pid, + { ...options, force: true }, + { onProgress: (phase) => phases.push(phase) }, + ); + expect(phases).toEqual(['graph-ready', 'done']); + expect(published.graphGeneration?.id).toBe(handoff.publicationId); + expect(await contents(published)).toContain('return 1'); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'parse-rebuild'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + } finally { + if (child.connected) child.send('exit'); + await exit; + } +}, 120_000); diff --git a/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts b/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts new file mode 100644 index 000000000..80bb5c913 --- /dev/null +++ b/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts @@ -0,0 +1,184 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer } from 'node:http'; +import { execFileSync, fork, spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { loadMeta } from '../../src/storage/repo-manager.js'; +import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; + +it.each(['CLI', 'CLI pinned heap', 'auto-sync', 'embeddings sync'] as const)( + '%s publishes after writer exit and resumes only unfinished HTTP embeddings', + async (frontend) => { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-cli-phases-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + const content = Array.from( + { length: 130 }, + (_, i) => `export function alpha${String(i).padStart(3, '0')}() { return ${i}; }`, + ).join('\n'); + await fs.writeFile(path.join(repo, 'a.ts'), content); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + let fail = true; + const accepted: string[] = []; + const observations: unknown[] = []; + const errors: unknown[] = []; + const server = createServer(async (request, response) => { + try { + const meta = await loadMeta(slot); + expect(meta?.graphGeneration).toBeDefined(); + expect(meta?.incrementalInProgress).toBeUndefined(); + expect(meta?.embeddingJob).toBeDefined(); + await expect(fs.stat(path.join(slot, 'analysis-publication.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + observations.push(meta?.graphGeneration?.id); + let body = ''; + for await (const chunk of request) body += chunk; + const input: string[] = JSON.parse(body).input; + if (fail && accepted.length >= 64) { + response.writeHead(503).end('outage'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })), + }), + ); + } catch (error) { + errors.push(error); + response.writeHead(500).end('fixture failure'); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing fixture port'); + const env = { + ...process.env, + // This fixture exercises automatic process-family admission and respawning. + GITNEXUS_MEMORY: '', + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_EMBEDDING_URL: `http://127.0.0.1:${address.port}/v1`, + GITNEXUS_EMBEDDING_MODEL: 'cli-phase-fixture', + GITNEXUS_EMBEDDING_DIMS: '4', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '8', + }; + const runCli = (embeddings: boolean, sync = false) => + new Promise<{ code: number | null; output: string }>((resolve, reject) => { + const child = spawn( + process.execPath, + [ + ...(sync ? ['--max-old-space-size=1024'] : []), + ...(frontend === 'CLI pinned heap' && !sync + ? ['--max-old-space-size=512', '--max-semi-space-size=16'] + : []), + fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url)), + ...(sync + ? ['embeddings', 'sync', repo] + : [ + 'analyze', + repo, + '--index-only', + '--skip-fts', + '--workers', + '1', + '--resource-budget', + '2048', + ...(embeddings ? ['--embeddings', '0'] : []), + ]), + ], + { env, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let output = ''; + const collect = (chunk: Buffer) => { + output = (output + chunk.toString()).slice(-32_768); + }; + child.stdout.on('data', collect); + child.stderr.on('data', collect); + child.once('error', reject); + child.once('close', (code) => resolve({ code, output })); + }); + const runWorker = async (embeddings: boolean) => { + const run = createAutoSyncAnalysisRunner({ + forkWorker: (_workerPath, execArgv) => + fork(fileURLToPath(new URL('../../dist/server/analyze-worker.js', import.meta.url)), [], { + execArgv, + env, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }), + }); + try { + await run( + repo, + { + embeddings, + embeddingsNodeLimit: 0, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, + workerPoolSize: 1, + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 2048 * 1024 ** 2, + totalMb: 2048, + availableCpus: 2, + workers: 1, + modelMemoryMb: 0, + }), + }, + 120_000, + ); + return { code: 0, output: '' }; + } catch (error) { + return { code: 1, output: (error as Error).message }; + } + }; + const run = frontend === 'auto-sync' ? runWorker : runCli; + const resume = frontend === 'embeddings sync' ? () => runCli(false, true) : run; + try { + const first = await run(true); + expect(first.code, first.output).toBe(1); + expect(errors).toEqual([]); + expect(accepted, first.output).toHaveLength(64); + expect(first.output).not.toContain('Repository indexed successfully'); + const graph = await loadMeta(slot); + expect(graph?.embeddingJob).toBeDefined(); + if (!graph?.graphPath) throw new Error('Expected owned graph'); + const inode = (await fs.stat(path.join(slot, graph.graphPath))).ino; + fail = false; + const retry = await resume(false); + expect(retry.code, retry.output).toBe(0); + if (frontend === 'CLI') + expect(retry.output.match(/Repository indexed successfully/g)).toHaveLength(1); + expect(accepted).toHaveLength(130); + expect(new Set(accepted).size).toBe(130); + expect(new Set(observations)).toEqual(new Set([graph.graphGeneration?.id])); + expect((await loadMeta(slot))?.embeddingJob).toBeUndefined(); + const repeat = await resume(true); + expect(repeat.code, repeat.output).toBe(0); + expect(accepted).toHaveLength(130); + await fs.writeFile(path.join(repo, 'a.ts'), content.replace('return 0;', 'return 999;')); + const changed = await run(true); + expect(changed.code, changed.output).toBe(0); + expect(accepted).toHaveLength(131); + const updated = await loadMeta(slot); + expect(updated?.graphGeneration?.id).not.toBe(graph.graphGeneration?.id); + expect(updated?.graphPath).toBe(graph.graphPath); + expect((await fs.stat(path.join(slot, graph.graphPath))).ino).toBe(inode); + expect(errors).toEqual([]); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + } + }, + 180_000, +); diff --git a/gitnexus/test/integration/analyze-http-embedding-phases.test.ts b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts new file mode 100644 index 000000000..2207a84ac --- /dev/null +++ b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts @@ -0,0 +1,153 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer } from 'node:http'; +import { execFileSync, fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { loadMeta } from '../../src/storage/repo-manager.js'; + +it('serves the published graph during an HTTP outage and resumes through /api/embed without repeating paid work', async () => { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-http-phases-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + await fs.writeFile( + path.join(repo, 'a.ts'), + Array.from( + { length: 130 }, + (_, i) => `export function alpha${String(i).padStart(3, '0')}() { return ${i}; }`, + ).join('\n'), + ); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + let fail = true; + let releaseOutage!: () => void; + const outage = new Promise((resolve) => { + releaseOutage = resolve; + }); + const accepted: string[] = []; + const provider = createServer(async (request, response) => { + let body = ''; + for await (const chunk of request) body += chunk; + const input: string[] = JSON.parse(body).input; + if (fail && accepted.length >= 64) { + await outage; + response.writeHead(503).end('outage'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ data: input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })) }), + ); + }); + await new Promise((resolve) => provider.listen(0, '127.0.0.1', resolve)); + const providerAddress = provider.address(); + if (!providerAddress || typeof providerAddress === 'string') throw new Error('No provider port'); + const child = fork( + fileURLToPath(new URL('../helpers/analysis-http-server.mjs', import.meta.url)), + [], + { + execArgv: ['--max-old-space-size=1024'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + env: { + ...process.env, + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_SERVER_ANALYZE_HEAP_MB: '512', + GITNEXUS_WORKER_POOL_SIZE: '1', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(256 * 1024 ** 2), + GITNEXUS_EMBEDDING_URL: `http://127.0.0.1:${providerAddress.port}/v1`, + GITNEXUS_EMBEDDING_MODEL: 'http-phase-fixture', + GITNEXUS_EMBEDDING_DIMS: '4', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '8', + }, + }, + ); + let output = ''; + const collect = (chunk: Buffer) => { + output = (output + chunk.toString()).slice(-32768); + }; + child.stdout?.on('data', collect); + child.stderr?.on('data', collect); + const exited = new Promise((resolve) => child.once('exit', resolve)); + const port = await new Promise((resolve, reject) => { + child.once('message', (message) => resolve((message as { port: number }).port)); + child.once('error', reject); + void exited.then((code) => reject(new Error(`Server exited ${code}: ${output}`))); + }); + const request = async (route: string, body?: unknown) => { + const response = await fetch(`http://127.0.0.1:${port}${route}`, { + method: body === undefined ? 'GET' : 'POST', + headers: { 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const value = (await response.json()) as Record; + expect(response.ok, JSON.stringify(value) + '\n' + output).toBe(true); + return value; + }; + const until = async (check: () => Promise) => { + const end = Date.now() + 90_000; + while (!(await check())) { + if (Date.now() > end) throw new Error(`Timed out: ${output}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }; + try { + const started = await request('/api/analyze', { path: repo, embeddings: true }); + const poll = () => request(`/api/analyze/${started.jobId}`); + await until(async () => (await poll()).status === 'analyzing'); + const graphQuery = request('/api/query?repo=repo', { + cypher: 'MATCH (n:Function) RETURN count(n) AS count', + }); + await until(async () => accepted.length === 64); + expect((await graphQuery).error).toBeUndefined(); + expect((await poll()).status).toBe('analyzing'); + expect((await loadMeta(slot))?.graphGeneration).toBeDefined(); + const keyword = await request('/api/search?repo=repo', { + query: 'alpha000', + mode: 'bm25', + enrich: false, + }); + expect(JSON.stringify(keyword)).toContain('alpha000'); + releaseOutage(); + await until(async () => (await poll()).status === 'failed'); + expect(accepted).toHaveLength(64); + fail = false; + const retry = await request('/api/embed?repo=repo', {}); + await until(async () => { + const job = await request(`/api/embed/${retry.jobId}`); + if (job.status === 'failed') throw new Error(JSON.stringify(job) + '\n' + output); + return job.status === 'complete'; + }); + expect(accepted).toHaveLength(130); + expect(new Set(accepted).size).toBe(130); + expect((await loadMeta(slot))?.embeddingJob).toBeUndefined(); + const semantic = await request('/api/search?repo=repo', { + query: 'function returns a number', + mode: 'semantic', + enrich: false, + }); + expect(JSON.stringify(semantic)).toContain('alpha'); + const paidInputs = accepted.length; + const generation = (await loadMeta(slot))?.graphGeneration; + const cached = await request('/api/analyze', { path: repo, embeddings: true }); + await until(async () => { + const job = await request(`/api/analyze/${cached.jobId}`); + if (job.status === 'failed') throw new Error(JSON.stringify(job) + '\n' + output); + return job.status === 'complete'; + }); + expect(accepted).toHaveLength(paidInputs); + expect((await loadMeta(slot))?.graphGeneration).toEqual(generation); + } finally { + releaseOutage(); + if (child.connected) child.send('shutdown'); + expect(await exited, output).toBe(0); + await new Promise((resolve) => provider.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + } +}, 180_000); diff --git a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts index 33db6b36b..3ff345d09 100644 --- a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts +++ b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts @@ -1,6 +1,6 @@ /** - * Exercise the CLI -> native staged DB -> durable checkpoint -> SIGKILL -> - * isolated recovery -> graph rebuild -> publication chain. The endpoint is + * Exercise a legacy native staged DB -> durable checkpoint -> SIGKILL -> + * current CLI recovery -> graph rebuild -> publication chain. The endpoint is * local and deterministic; request text is the billing/reuse oracle. */ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; @@ -16,11 +16,22 @@ const DIMS = 8; const NODE_COUNT = 5_128; const DEADLINE = process.env.CI ? 180_000 : 120_000; const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const adapterUrl = pathToFileURL(path.join(packageRoot, 'src/core/lbug/lbug-adapter.ts')).href; +const nativeUrl = pathToFileURL(path.join(packageRoot, 'src/core/lbug/lbug-config.ts')).href; +const analysisUrl = pathToFileURL( + path.join( + packageRoot, + process.env.GITNEXUS_E2E_CLI === 'dist' + ? 'dist/core/run-analyze.js' + : 'src/core/run-analyze.ts', + ), +).href; interface CheckpointMeta { repoPath: string; stats?: { embeddings?: number }; + embeddingJob?: unknown; + graphPath?: string; + vectorGeneration?: { identity: string; artifactIdentity?: string }; embeddingCheckpoint?: { nodesProcessed: number; chunksProcessed: number; @@ -42,6 +53,7 @@ let endpoint: string; let submitted: string[] = []; let completed: string[] = []; let durableTexts: string[] = []; +let lastOutput = ''; const activeWindowCompleted: string[] = []; let held: string[] = []; let stopAfterCheckpoint = false; @@ -81,21 +93,41 @@ function cliEnv(repo: string): NodeJS.ProcessEnv { }; } -function runAnalyze(repo: string, flags: string[] = [], onOutput?: (output: string) => void): Run { +function runAnalyze( + repo: string, + flags: string[] = [], + onOutput?: (output: string) => void, + legacyFixture = false, +): Run { let output = ''; + // Generate the old checkpoint format through the retained legacy core path. + // Every recovery below still enters through the current public CLI. + const legacyScript = ` + const { runFullAnalysis } = await import(${JSON.stringify(analysisUrl)}); + await runFullAnalysis(${JSON.stringify(repo)}, { + force: true, embeddings: true, skipSkills: true, skipAgentsMd: true, + skipFts: true, workerPoolSize: 1, skipNativeCloseOnExit: true, + }, { onProgress() {}, onLog: console.log }); + process.exit(0); + `; + // A file entry keeps --input-type out of the parser workers' inherited argv. + const legacyEntry = path.join(root, 'legacy-analyze.mjs'); + if (legacyFixture) fs.writeFileSync(legacyEntry, legacyScript); const child = spawn( process.execPath, - [ - ...CLI_SPAWN_PREFIX, - 'analyze', - repo, - '--no-share', - '--skip-skills', - '--skip-fts', - '--workers', - '1', - ...flags, - ], + legacyFixture + ? ['--import', tsxLoaderUrl(), legacyEntry] + : [ + ...CLI_SPAWN_PREFIX, + 'analyze', + repo, + '--no-share', + '--skip-skills', + '--skip-fts', + '--workers', + '1', + ...flags, + ], { cwd: repo, env: cliEnv(repo), stdio: ['ignore', 'pipe', 'pipe'] }, ); running.add(child); @@ -124,6 +156,7 @@ async function successfulAnalyze(repo: string, flags: string[] = []): Promise row.nodeId)).size).toBe(expectedNodes); expect(meta.stats?.embeddings).toBe(rows.length); @@ -208,6 +257,19 @@ function expectPublishedComplete(repo: string, expectedNodes: number): void { ).toEqual([]); } +function expectLegacyReuse(texts: string[], unfinishedInputs: number): void { + // Legacy checkpoints record the provider but not chunk size/overlap. The + // complete single-chunk groups are provably reusable; the long group's + // historical input boundaries cannot be certified and must be explained. + const unverified = durableTexts.filter((text) => text.includes('longRecoverable')); + const reusable = new Set(durableTexts.filter((text) => !text.includes('longRecoverable'))); + expect(texts.filter((text) => reusable.has(text))).toEqual([]); + expect(unverified.length).toBeGreaterThan(1); + for (const text of unverified) expect(texts).toContain(text); + expect(texts).toHaveLength(unfinishedInputs + unverified.length); + expect(lastOutput).toContain('Legacy multi-chunk input provenance is unknown'); +} + beforeAll(async () => { if (process.platform === 'win32') return; root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-staged-recovery-e2e-')); @@ -219,7 +281,8 @@ beforeAll(async () => { { length: NODE_COUNT - 1 }, (_, index) => `export function recoverable${index}() { return ${index}; }`, ); - // Multi-chunk nodes must be reused as a complete group, including their tail. + // Missing legacy input provenance must invalidate the whole multi-chunk + // group, while retaining every provably compatible single-chunk group. const longFunction = `export function longRecoverable() {\n${Array.from({ length: 80 }, (_, index) => ` // retained chunk marker ${index} ${'x'.repeat(80)}`).join('\n')}\n return 42;\n}`; fs.writeFileSync( path.join(stoppedRepo, 'functions.ts'), @@ -284,7 +347,7 @@ beforeAll(async () => { const gate = new Promise((resolve) => { gateResolve = resolve; }); - const first = runAnalyze(stoppedRepo, ['--force', '--embeddings']); + const first = runAnalyze(stoppedRepo, [], undefined, true); await Promise.race([ gate, first.done.then((result) => { @@ -328,30 +391,27 @@ describe.skipIf(process.platform === 'win32')( ['plain', []], ['forced', ['--force', '--embeddings']], ] as const)( - '%s retry bills only unfinished groups and publishes an honest count', + '%s retry reuses verified groups and explains unverified legacy input', async (name, flags) => { const repo = cloneStoppedRepo(name); const texts = await successfulAnalyze(repo, [...flags]); - const durable = new Set(durableTexts); - expect(texts.filter((text) => durable.has(text))).toEqual([]); + expectLegacyReuse(texts, 128); for (const text of held) expect(texts).toContain(text); for (const text of activeWindowCompleted) expect(texts).toContain(text); - expect(texts.length).toBe(128); expectPublishedComplete(repo, NODE_COUNT); }, DEADLINE, ); it( - 'manual checkpoint opt-out completes a staged retry without resubmitting durable chunks', + 'manual checkpoint opt-out retains verified staged groups', async () => { const repo = cloneStoppedRepo('manual-checkpoint-opt-out'); const previous = process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = '0'; try { const texts = await successfulAnalyze(repo, ['--force', '--embeddings']); - expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(texts.length).toBe(128); + expectLegacyReuse(texts, 128); expectPublishedComplete(repo, NODE_COUNT); } finally { if (previous === undefined) delete process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; @@ -384,8 +444,7 @@ describe.skipIf(process.platform === 'win32')( expect(readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile).toBe(source); expect(fs.existsSync(path.join(repo, '.gitnexus', source))).toBe(true); const finalTexts = await successfulAnalyze(repo); - expect(finalTexts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(finalTexts.length).toBe(128); + expectLegacyReuse(finalTexts, 128); expectPublishedComplete(repo, NODE_COUNT); }, DEADLINE * 2, @@ -412,8 +471,7 @@ describe.skipIf(process.platform === 'win32')( true, ); expect(texts.some((text) => text.includes('recoverable6'))).toBe(false); - expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(texts.length).toBe(129); + expectLegacyReuse(texts, 129); expectPublishedComplete(repo, NODE_COUNT - 1); }, DEADLINE, diff --git a/gitnexus/test/integration/embedding-job-restart.test.ts b/gitnexus/test/integration/embedding-job-restart.test.ts new file mode 100644 index 000000000..4e4367769 --- /dev/null +++ b/gitnexus/test/integration/embedding-job-restart.test.ts @@ -0,0 +1,157 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; + +const execute = promisify(execFile); +const helper = fileURLToPath(new URL('../helpers/embedding-restart-runner.mjs', import.meta.url)); +const boundaries = [ + 'intent-before', + 'intent-after', + 'appendBatch-before', + 'appendBatch-after', + 'checkpointUnit-before', + 'checkpointUnit-after', + 'cursor-before', + 'cursor-after', + 'seal-after', + 'pointer-after', +] as const; + +it.each(boundaries)( + 'resumes a real process exit at %s without replaying acknowledged inputs', + async (boundary) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-restart-')); + const env = { + ...process.env, + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + }; + let store: VectorStore | undefined; + try { + await saveMeta(root, { + repoPath: root, + lastCommit: 'fixture', + indexedAt: 'fixture', + graphPath: `lbug.generation.${'a'.repeat(32)}`, + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + schemaFingerprint: 'fixture', + parseCacheIdentity: { base: 'fixture' }, + scopeExtractionReceipt: 1, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'fixture', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: process.versions.modules, + libc: 'fixture', + }, + invokedArtifact: { path: helper, digest: 'fixture' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'fixture', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'fixture', + }, + }, + }); + await expect( + execute(process.execPath, [helper, root, boundary], { env }), + ).rejects.toMatchObject({ code: 73 }); + await execute(process.execPath, [helper, root, 'resume'], { env }); + const readInputs = async ( + phase: string, + ): Promise> => { + const text = await fs + .readFile(path.join(root, `${phase}.inputs.jsonl`), 'utf8') + .catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return ''; + throw error; + }); + return text + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + }; + const first = await readInputs(boundary); + const resumed = await readInputs('resume'); + expect(resumed.every(({ node }) => node === null || Number(node.slice(1)) >= 64)).toBe(true); + if (boundary === 'seal-after' || boundary === 'pointer-after') expect(resumed).toEqual([]); + const paid = new Map(); + for (const input of [...first, ...resumed]) { + const entry = paid.get(input.hash) ?? { node: input.node, count: 0 }; + entry.count++; + paid.set(input.hash, entry); + } + expect( + new Set([...paid.values()].map(({ node }) => node).filter((node) => node !== null)), + ).toEqual( + new Set(Array.from({ length: 130 }, (_, index) => `n${String(index).padStart(5, '0')}`)), + ); + for (const { node, count } of paid.values()) { + // Only n00064 has interior slices without a name, all in the active unit. + const mayReplay = + node === null || (Number(node.slice(1)) >= 64 && Number(node.slice(1)) < 128); + expect(count).toBeLessThanOrEqual(mayReplay ? 2 : 1); + } + const metadata = await loadMeta(root); + const generation = metadata?.vectorGeneration; + if (!generation) throw new Error('Restart did not publish vectors'); + expect(metadata.embeddingJob).toBeUndefined(); + expect(generation.nodes).toBe(130); + store = await VectorStore.open({ + directory: path.join( + root, + 'embeddings', + generation.artifactIdentity ?? generation.identity, + ), + identity: generation.artifactIdentity ?? generation.identity, + dimensions: 4, + mode: 'read', + bufferPoolBytes: 64 * 1024 ** 2, + threads: 1, + batchBytes: 64 * 1024, + }); + let totalChunks = 0; + for (let offset = 0; offset < 130; offset += 64) { + const ids = Array.from( + { length: Math.min(64, 130 - offset) }, + (_, index) => `Function:a.ts:n${String(offset + index).padStart(5, '0')}`, + ); + const complete = await store.completedNodes(ids); + expect(complete.size).toBe(ids.length); + for (const [id, receipt] of complete) { + const chunks = await store.readChunks(receipt, 0, receipt.chunks); + expect(chunks.map(({ chunkIndex }) => chunkIndex)).toEqual( + Array.from({ length: receipt.chunks }, (_, index) => index), + ); + if (id.endsWith('n00064')) expect(chunks.length).toBeGreaterThan(1); + totalChunks += chunks.length; + } + } + expect(totalChunks).toBe(generation.chunks); + } finally { + await store?.close(); + await fs.rm(root, { recursive: true, force: true }); + } + }, + 120_000, +); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index f80c6810b..c8d41eaa0 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -246,6 +246,25 @@ describe('rebased pending vector inputs', () => { }); describe('bounded durable embedding runner', () => { + it('completes copied groups together at the unit boundary without provider requests', async () => { + const fixture = await runnerFixture(64); + const finish = vi.spyOn(store, 'finishNodes'); + const result = await fixture.run({ + reuse: { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 }; + }, + async readChunks() { + return [chunk(0)]; + }, + }, + }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(result.state).toMatchObject({ reusedNodes: 64, completedNodes: 0, chunks: 64 }); + expect(finish).toHaveBeenCalledTimes(1); + }); + it('refuses a symlinked native shadow sidecar before opening the artifact', async () => { const options = await setup(); await store.close(); @@ -677,6 +696,50 @@ describe('bounded durable embedding runner', () => { }); describe('separate durable vector artifacts', () => { + it('verifies a complete unit with bounded native queries instead of one query per node', async () => { + await setup(); + const nodes = Array.from({ length: 64 }, (_, i) => ({ + nodeId: `Function:a:batch${i}`, + contentHash: `hash${i}`, + })); + await store.beginNodes(nodes); + await store.appendBatch(nodes.map((node) => ({ ...node, ...chunk(0) }))); + const execute = vi.spyOn(lbug.Connection.prototype, 'execute'); + const receipts = await store.finishNodes(nodes.map((node) => ({ ...node, chunks: 1 }))); + expect(receipts).toHaveLength(64); + expect(receipts.every((receipt) => receipt.complete)).toBe(true); + expect(execute.mock.calls.length).toBeLessThanOrEqual(3); + execute.mockRestore(); + await store.checkpointUnit(receipts); + }); + + it('rejects a cross-node chunk mismatch before completing any group in a unit', async () => { + const options = await setup(); + const nodes = [0, 1].map((i) => ({ nodeId: `Function:a:group${i}`, contentHash: `hash${i}` })); + await store.beginNodes(nodes); + await store.appendBatch(nodes.map((node) => ({ ...node, ...chunk(0) }))); + await store.close(); + store = undefined; + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query( + "MATCH (e:CodeEmbedding {id:'Function:a:group1:0'}) SET e.nodeId='Function:a:group0'", + ); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } finally { + await closeLbugConnection(handle); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await expect(store.finishNodes(nodes.map((node) => ({ ...node, chunks: 1 })))).rejects.toThrow( + 'invalid chunk set', + ); + for (const node of nodes) + expect(await store.reusable(node.nodeId, node.contentHash)).toBeNull(); + }); + it('writes a multi-node unit and verifies a large node in bounded batches', async () => { const options = await setup(); const nodes = [0, 1].map((i) => ({ nodeId: `Function:a:node${i}`, contentHash: 'hash' })); diff --git a/gitnexus/test/integration/lbug-readonly-init.test.ts b/gitnexus/test/integration/lbug-readonly-init.test.ts index 1d79ac39c..6682d9d3e 100644 --- a/gitnexus/test/integration/lbug-readonly-init.test.ts +++ b/gitnexus/test/integration/lbug-readonly-init.test.ts @@ -6,11 +6,40 @@ * filesystems such as Docker :ro bind mounts. */ import fs from 'fs/promises'; +import lbug from '@ladybugdb/core'; import { it, expect } from 'vitest'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; import { _initLockPathForTest } from '../../src/core/lbug/lbug-adapter.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; withTestLbugDB('lbug-readonly-init', (handle) => { + it('closes a read-only graph without leaving a checkpoint marker that blocks the next reader', async () => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.closeLbug(); + await adapter.withLbugDb(handle.dbPath, () => adapter.executeQuery('RETURN 1 AS value'), { + readOnly: true, + }); + expect(await adapter.tryFlushWAL()).toBe(false); + await adapter.closeLbug(); + // Open through the raw bounded reader, so the adapter cannot hide a bad + // read-only close by recovering it through a subsequent writable open. + const next = await openLbugConnection(lbug, handle.dbPath, { + readOnly: true, + bufferPoolBytes: 64 * 1024 ** 2, + threads: 1, + }); + try { + const result = await next.conn.query('RETURN 1 AS value'); + const cursors = Array.isArray(result) ? result : [result]; + try { + expect(await cursors[0].getAll()).toEqual([{ value: 1 }]); + } finally { + for (const cursor of cursors) cursor.close(); + } + } finally { + await closeLbugConnection(next, { strict: true, databasePath: handle.dbPath }); + } + }); it('read-only open never creates lbug.init.lock on disk', async () => { const { dbPath } = handle; const lockPath = _initLockPathForTest(dbPath); diff --git a/gitnexus/test/integration/legacy-vector-export.test.ts b/gitnexus/test/integration/legacy-vector-export.test.ts new file mode 100644 index 000000000..4d52999c4 --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-export.test.ts @@ -0,0 +1,262 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, expect, it, vi } from 'vitest'; +import { exportLegacyVectors } from '../../src/core/lbug/legacy-vector-export.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { LegacyVectorSource } from '../../src/core/lbug/legacy-vector-source.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { readLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; + +const roots: string[] = []; +const pool = 32 * 1024 * 1024; +const identity = { provider: 'local', model: 'historical-model', dimensions: 4 }; +const config = resolveEmbeddingConfig({ threads: 1 }); +const node: EmbeddableNode = { + id: 'Const:a.ts:value', + name: 'value', + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, +}; +afterEach(async () => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture(rows = 1) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "gn-legacy-export-'")); + roots.push(root); + const file = path.join(root, 'lbug'); + const handle = await openLbugConnection(lbug, file, { bufferPoolBytes: pool, threads: 1 }); + const query = async (sql: string): Promise[]> => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }; + await query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ); + await query('CREATE NODE TABLE File (id STRING, content STRING, PRIMARY KEY(id))'); + await query( + "CREATE (:File {id:'File:a.ts',content:'Graph content must stay out of the export'})", + ); + const hash = contentHashForNode(node, config); + if (rows) + await query( + `UNWIND range(0, ${rows - 1}) AS i CREATE (:CodeEmbedding {id:'Const:a.ts:value:' + CAST(i AS STRING),nodeId:'Const:a.ts:value',chunkIndex:i,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'${hash}'})`, + ); + await query('CHECKPOINT'); + const options = { + metadataDirectory: root, + query, + provenance: { schemaFingerprint: 'schema', recordedIdentity: identity }, + excluded: [] as string[], + bufferPoolBytes: pool, + threads: 1, + }; + return { + root, + file, + query, + hash, + options, + close: () => closeLbugConnection(handle, { strict: true, databasePath: file }), + }; +} + +it('exports only the vector table and retains a compatible complete group without provider work', async () => { + const f = await fixture(); + const before = await fs.stat(f.file, { bigint: true }); + const copy = vi.spyOn(fs, 'copyFile'); + let store: VectorStore | undefined, source: LegacyVectorSource | undefined; + try { + const id = await exportLegacyVectors(f.options); + expect(id).toMatch(/^[a-f0-9]{64}$/); + const receipt = await readLegacyEmbeddingSource(f.root, id!); + expect(receipt.recordedIdentity).toEqual(identity); + expect(copy).not.toHaveBeenCalled(); + expect((await fs.stat(f.file, { bigint: true })).ino).toBe(before.ino); + expect(await f.query('MATCH (n:File) RETURN n.content AS content')).toEqual([ + { content: 'Graph content must stay out of the export' }, + ]); + const exported = await openLbugConnection(lbug, receipt.path, { + readOnly: true, + bufferPoolBytes: pool, + threads: 1, + }); + try { + const result = await exported.conn.query('CALL show_tables() RETURN name'); + const cursor = Array.isArray(result) ? result[0] : result; + try { + expect(await cursor.getAll()).toEqual([{ name: 'CodeEmbedding' }]); + } finally { + cursor.close(); + } + } finally { + await closeLbugConnection(exported, { strict: true, databasePath: receipt.path }); + } + await fs.mkdir(path.join(f.root, 'reuse')); + store = await VectorStore.open({ + directory: path.join(f.root, 'reuse'), + identity: 'b'.repeat(64), + dimensions: 4, + mode: 'create', + bufferPoolBytes: pool, + threads: 1, + batchBytes: 65536, + }); + source = await LegacyVectorSource.open( + { + path: receipt.path, + sourceIdentity: receipt.identity, + recordedIdentity: receipt.recordedIdentity, + expectedIdentity: identity, + expectedInputIdentity: 'c'.repeat(64), + config, + bufferPoolBytes: pool, + threads: 1, + batchBytes: 65536, + }, + store, + receipt.excludedIds, + ); + const group = await source.reusable(node.id, f.hash, node); + expect(group?.chunks).toBe(1); + expect(await source.readChunks(group!, 0, 1)).toEqual([ + { chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0, 0, 0] }, + ]); + } finally { + await source?.close(); + await store?.close(); + await f.close(); + } +}); + +it('keeps vector payloads out of JavaScript while transferring a larger table', async () => { + const f = await fixture(4096); + let largestResult = 0; + try { + const id = await exportLegacyVectors({ + ...f.options, + query: async (sql) => { + const rows = await f.query(sql); + largestResult = Math.max(largestResult, rows.length); + expect(rows.every((row) => !Object.hasOwn(row, 'embedding'))).toBe(true); + return rows; + }, + }); + expect(id).toBeDefined(); + expect(largestResult).toBe(7); + expect(await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count')).toEqual([ + { count: 4096 }, + ]); + } finally { + await f.close(); + } +}); + +it('does not invent missing historical provider or input provenance', async () => { + const f = await fixture(); + try { + const id = await exportLegacyVectors({ + ...f.options, + provenance: { schemaFingerprint: 'schema' }, + excluded: [node.id], + }); + const receipt = await readLegacyEmbeddingSource(f.root, id!); + expect(receipt.recordedIdentity).toBeUndefined(); + expect(receipt.recordedInputIdentity).toBeUndefined(); + const ids = []; + for await (const id of receipt.excludedIds()) ids.push(id); + expect(ids).toEqual([node.id]); + } finally { + await f.close(); + } +}); + +it('does not create an artifact for an empty vector table', async () => { + const f = await fixture(0); + try { + expect(await exportLegacyVectors(f.options)).toBeUndefined(); + expect( + (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), + ).toEqual([]); + } finally { + await f.close(); + } +}); + +it('rejects mismatched historical dimensions before exporting', async () => { + const f = await fixture(); + try { + await expect( + exportLegacyVectors({ + ...f.options, + provenance: { + schemaFingerprint: 'schema', + recordedIdentity: { ...identity, dimensions: 8 }, + }, + }), + ).rejects.toThrow('schema differs'); + expect( + (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), + ).toEqual([]); + } finally { + await f.close(); + } +}); + +it('refuses publication when the source count changes during the transfer', async () => { + const f = await fixture(); + let counts = 0; + try { + await expect( + exportLegacyVectors({ + ...f.options, + query: async (sql) => { + if (sql.includes('count(e)') && ++counts === 2) return [{ count: 2 }]; + return f.query(sql); + }, + }), + ).rejects.toThrow('source changed'); + expect( + (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), + ).toEqual([]); + expect((await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count'))[0].count).toBe(1); + } finally { + await f.close(); + } +}); + +it('cancels after the native export returns without publishing or mutating its source', async () => { + const f = await fixture(); + const abort = new AbortController(); + try { + await expect( + exportLegacyVectors({ + ...f.options, + signal: abort.signal, + query: async (sql) => { + const result = await f.query(sql); + if (sql.startsWith('COPY')) abort.abort(new Error('cancel migration')); + return result; + }, + }), + ).rejects.toThrow('cancel migration'); + expect((await fs.readdir(f.root)).filter((name) => /generation|\.csv$/.test(name))).toEqual([]); + expect((await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count'))[0].count).toBe(1); + } finally { + await f.close(); + } +}); diff --git a/gitnexus/test/unit/analysis-embedding-phase.test.ts b/gitnexus/test/unit/analysis-embedding-phase.test.ts new file mode 100644 index 000000000..1e40299dd --- /dev/null +++ b/gitnexus/test/unit/analysis-embedding-phase.test.ts @@ -0,0 +1,133 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; + +const mocks = vi.hoisted(() => ({ load: vi.fn(), save: vi.fn(), prepared: vi.fn(), run: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: mocks.load, + saveMeta: mocks.save, +})); +vi.mock('../../src/core/analysis-publication.js', () => ({ + readPreparedAnalysis: mocks.prepared, + beginInPlaceAnalysis: vi.fn(), + prepareAnalysisPublication: vi.fn(), + publishPreparedAnalysis: vi.fn(), +})); +vi.mock('../../src/core/embeddings/embedding-job.js', () => ({ + runPublishedEmbeddingJob: mocks.run, +})); +vi.mock('../../src/core/embeddings/http-client.js', () => ({ isHttpMode: () => true })); +vi.mock('../../src/core/embeddings/embedding-identity.js', () => ({ + resolveEmbeddingIdentity: () => ({ provider: 'http:test', model: 'fixture', dimensions: 4 }), +})); +import { runAnalysisEmbeddingPhase } from '../../src/core/analysis-embedding-phase.js'; + +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 2048 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, +}); +const metadata = { + repoPath: '/repo', + lastCommit: 'a'.repeat(40), + indexedAt: '', + stats: { nodes: 3, embeddings: 1 }, + graphGeneration: { id: 'b'.repeat(32) }, + embeddingJob: { identity: 'retained' }, +}; +const options = () => ({ + metadataDirectory: '/repo/.gitnexus', + graphPath: '/repo/.gitnexus/graph.lbug', + resources, + callbacks: { onProgress: vi.fn(), onGraphPublished: vi.fn(async () => {}) }, +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.load.mockResolvedValue(metadata); + mocks.prepared.mockResolvedValue(undefined); + mocks.run.mockResolvedValue({ phase: 'ready', metadata }); +}); + +describe('shared embedding phase publication contract', () => { + it('awaits reader refresh before admitting provider work', async () => { + const input = options(); + let visible!: () => void; + input.callbacks.onGraphPublished.mockImplementationOnce( + () => + new Promise((resolve) => { + visible = resolve; + }), + ); + const pending = runAnalysisEmbeddingPhase(input); + await vi.waitFor(() => expect(input.callbacks.onGraphPublished).toHaveBeenCalledOnce()); + expect(mocks.run).not.toHaveBeenCalled(); + visible(); + await expect(pending).resolves.toBe(metadata); + expect(mocks.run).toHaveBeenCalledOnce(); + }); + + it('does not report incomplete vector work as complete or clear its durable metadata', async () => { + mocks.run.mockResolvedValue({ phase: 'embedding', metadata }); + const input = options(); + await expect(runAnalysisEmbeddingPhase(input)).rejects.toThrow('work remains incomplete'); + expect(mocks.save).not.toHaveBeenCalled(); + expect(input.callbacks.onProgress).toHaveBeenLastCalledWith( + 'embeddings', + 98, + expect.any(String), + expect.objectContaining({ graphPhase: 'ready', embeddingProgress: { phase: 'failed' } }), + ); + }); + + it('keeps the last measured count and pending work when provider or final verification fails', async () => { + mocks.run.mockRejectedValue(new Error('Could not verify persisted embedding count')); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow( + 'Could not verify persisted embedding count', + ); + expect(mocks.save).not.toHaveBeenCalled(); + expect(metadata.stats.embeddings).toBe(1); + expect(metadata.embeddingJob.identity).toBe('retained'); + }); + + it('rejects a superseded graph before reader refresh or paid work', async () => { + const input = options(); + await expect( + runAnalysisEmbeddingPhase({ ...input, expectedGeneration: 'c'.repeat(32) }), + ).rejects.toThrow('Graph generation changed'); + expect(mocks.run).not.toHaveBeenCalled(); + expect(input.callbacks.onGraphPublished).not.toHaveBeenCalled(); + }); + + it.each(['partial', 'unverified-count'])( + 'rejects a foreign %s checkpoint before provider work', + async (kind) => { + mocks.load.mockResolvedValue({ + ...metadata, + embeddingCheckpoint: { + provider: 'http:other', + model: 'other', + dimensions: 4, + kind, + at: '', + nodesProcessed: 1, + totalNodes: 3, + chunksProcessed: 1, + pendingNodeIds: ['n2'], + }, + }); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow( + /different provider|Cannot resume/, + ); + expect(mocks.run).not.toHaveBeenCalled(); + expect(mocks.save).not.toHaveBeenCalled(); + }, + ); + + it('refuses prepared or malformed graph receipts without touching embeddings', async () => { + mocks.prepared.mockResolvedValue({ id: 'a'.repeat(32) }); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow('awaits publication'); + mocks.prepared.mockRejectedValue(new Error('Malformed publication receipt')); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow('Malformed'); + expect(mocks.run).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts new file mode 100644 index 000000000..8a0e9f88e --- /dev/null +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -0,0 +1,101 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { + analysisMayEmbed, + resolveHostAnalysisResources, +} from '../../src/core/analysis-host-resources.js'; + +const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), placement: vi.fn() })); +vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ + effectiveRamBytes: () => 32768 * 1024 ** 2, +})); +vi.mock('node:os', () => ({ default: { availableParallelism: () => 16 } })); +vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: mocks.loadMeta, + getStoragePaths: (repo: string, branch: string | null) => ({ + metaPath: `${repo}/index/${branch ?? 'workspace'}/gitnexus.json`, + }), +})); +vi.mock('../../src/storage/branch-index.js', () => ({ resolveBranchPlacement: mocks.placement })); + +beforeEach(() => { + vi.clearAllMocks(); + for (const key of [ + 'GITNEXUS_EMBEDDING_URL', + 'GITNEXUS_EMBEDDING_MODEL', + 'GITNEXUS_WORKER_POOL_SIZE', + 'GITNEXUS_WORKER_HEAP_MB', + 'GITNEXUS_EMBEDDING_THREADS', + 'GITNEXUS_LBUG_BUFFER_POOL_SIZE', + ]) + vi.stubEnv(key, undefined); + mocks.loadMeta.mockResolvedValue(null); + mocks.placement.mockResolvedValue({ branch: 'release' }); +}); +afterEach(() => vi.unstubAllEnvs()); + +it('reserves local model capacity only for invocations that may embed', () => { + expect(resolveHostAnalysisResources({ includeEmbeddings: false }).embedding.modelBytes).toBe(0); + expect(resolveHostAnalysisResources({ includeEmbeddings: true }).embedding.modelBytes).toBe( + 2048 * 1024 ** 2, + ); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'http://localhost:1234/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'remote'); + expect(resolveHostAnalysisResources({ includeEmbeddings: true }).embedding.modelBytes).toBe(0); +}); + +it('divides the host allowance across concurrent analyses', () => { + const one = resolveHostAnalysisResources({ includeEmbeddings: false }); + const four = resolveHostAnalysisResources({ includeEmbeddings: false, concurrency: 4 }); + expect(four.totalBytes * 4).toBe(one.totalBytes); + expect(four.parse.nativeThreads * 4).toBe(one.parse.nativeThreads); + expect(four.parse.workers).toBeLessThanOrEqual(3); +}); + +it('honors bounded worker, pool, and embedding-thread overrides', () => { + vi.stubEnv('GITNEXUS_WORKER_POOL_SIZE', '8'); + vi.stubEnv('GITNEXUS_WORKER_HEAP_MB', '512'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(256 * 1024 ** 2)); + vi.stubEnv('GITNEXUS_EMBEDDING_THREADS', '8'); + const budget = resolveHostAnalysisResources({ includeEmbeddings: true }); + expect(budget.parse.workers).toBe(8); + expect(budget.parse.workerHeapMb).toBe(512); + expect(budget.parse.graphPoolBytes).toBe(256 * 1024 ** 2); + expect(budget.embedding.nativeThreads).toBe(8); +}); + +it.each(['0', '-1', '1.5', 'NaN'])( + 'rejects an unbounded or malformed native pool override %s', + (value) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', value); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'positive safe integer', + ); + }, +); + +it('honors explicit heap-only overcommit but rejects a conflicting family ceiling', () => { + expect( + resolveHostAnalysisResources({ includeEmbeddings: false, mainHeapMb: 40000 }).mainHeapMb, + ).toBe(40000); + expect(() => + resolveHostAnalysisResources({ includeEmbeddings: false, mainHeapMb: 40000, totalMb: 24576 }), + ).toThrow('cannot admit'); +}); + +it.each([{ embeddingJob: {} }, { embeddingCheckpoint: {} }, { stats: { embeddings: 1 } }])( + 'reserves retry/forced model work from the exact branch metadata', + async (metadata) => { + mocks.loadMeta.mockResolvedValue(metadata); + expect(await analysisMayEmbed('/repo', { force: true, branch: 'release' })).toBe(true); + expect(mocks.placement).toHaveBeenCalledWith('/repo', 'release'); + expect(mocks.loadMeta).toHaveBeenCalledWith('/repo/index/release'); + }, +); + +it('keeps plain preservation free of model work and lets explicit drop abandon retries', async () => { + mocks.loadMeta.mockResolvedValue({ stats: { embeddings: 100 } }); + expect(await analysisMayEmbed('/repo', {})).toBe(false); + mocks.loadMeta.mockResolvedValue({ embeddingJob: {} }); + expect(await analysisMayEmbed('/repo', { dropEmbeddings: true })).toBe(false); + expect(await analysisMayEmbed('/repo', { dropEmbeddings: true, embeddings: true })).toBe(true); +}); diff --git a/gitnexus/test/unit/analysis-parse-cache.test.ts b/gitnexus/test/unit/analysis-parse-cache.test.ts index 1de1aac55..4bba90e51 100644 --- a/gitnexus/test/unit/analysis-parse-cache.test.ts +++ b/gitnexus/test/unit/analysis-parse-cache.test.ts @@ -67,6 +67,7 @@ const options = (keys: string[] = [], complete = true) => ({ storagePath: root, metadataDirectory: root, cacheRoot: root, + publishedGeneration: generation, collectBranchKeys: async () => ({ keys: new Set(keys), complete }), log: vi.fn(), }); @@ -91,6 +92,16 @@ async function mutateReceipt(change: (payload: Record) => void) await fs.writeFile(file, JSON.stringify(envelope)); } +it('rejects a checksummed selection that repeats one key instead of including another', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys = new Set([a, b]); + await stageAnalysisParseCache(root, generation, cache); + await mutateReceipt((payload) => { + payload.used = [a, a]; + }); + await expect(loadAnalysisParseCache(root, metadata([a, b]))).rejects.toThrow('selection differs'); +}); + it('hands off keys without copying live shards, then merges a newer sibling publication', async () => { await live([ [a, 1], @@ -124,6 +135,24 @@ it('hands off keys without copying live shards, then merges a newer sibling publ expect((await loadParseCacheChunk(published, a))?.[0].fileCount).toBe(11); expect((await loadParseCacheChunk(published, c))?.[0].fileCount).toBe(3); expect((await loadParseCacheChunk(published, d))?.[0].fileCount).toBe(4); + await expect(fs.stat(path.join(root, `analysis-cache-${generation}.json`))).rejects.toMatchObject( + { code: 'ENOENT' }, + ); +}); + +it('retains its receipt when cache publication fails', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + await persistParseCacheChunk(cache, a, [result(2)]); + await stageAnalysisParseCache(root, generation, cache); + vi.spyOn(fs, 'rename').mockRejectedValueOnce(new Error('cache publication interrupted')); + await expect(publishAnalysisParseCache(await restored(metadata([a])), options())).rejects.toThrow( + 'cache publication interrupted', + ); + await expect( + fs.stat(path.join(root, `analysis-cache-${generation}.json`)), + ).resolves.toBeDefined(); + expect((await loadParseCacheChunk(await restored(metadata([a])), a))?.[0].fileCount).toBe(2); }); it('flushes memory fallback shards before the child can hand off its cache', async () => { diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index b9dfbedd4..e151df79b 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -8,6 +8,34 @@ const MIB = 1024 * 1024; const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; describe('analysis resource admission', () => { + it('preserves an explicit heap override without inflating automatic native allocations', () => { + const host = { ...request, effectiveMemoryBytes: 7500 * MIB, modelMemoryMb: 0 }; + const automatic = resolveAnalysisResourceBudget(host); + const budget = resolveAnalysisResourceBudget({ + ...host, + mainHeapMb: 12000, + allowHeapOvercommit: true, + }); + expect(budget.mainHeapMb).toBe(12000); + expect(budget.totalBytes).toBeGreaterThan(host.effectiveMemoryBytes); + expect(budget.parse.graphPoolBytes).toBe(automatic.parse.graphPoolBytes); + expect(budget.embedding.queueBytes).toBe(automatic.embedding.queueBytes); + expect(budget.headroomBytes).toBe(automatic.headroomBytes); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => resolveAnalysisResourceBudget({ ...host, mainHeapMb: 12000 })).toThrow( + 'cannot admit', + ); + expect(() => + resolveAnalysisResourceBudget({ + ...host, + mainHeapMb: 12000, + allowHeapOvercommit: true, + totalMb: 5625, + }), + ).toThrow('cannot admit'); + }); + it('charges a retained supervisor once per phase, independently of worker thread count', () => { const fixed = { ...request, diff --git a/gitnexus/test/unit/analyze-api.test.ts b/gitnexus/test/unit/analyze-api.test.ts index 076e86b70..19355c978 100644 --- a/gitnexus/test/unit/analyze-api.test.ts +++ b/gitnexus/test/unit/analyze-api.test.ts @@ -591,109 +591,9 @@ describe('POST /api/embed records the embedding count it measured', () => { }); }); -/** - * Wiring guard for the route. Everything the helpers DECIDE is pinned - * behaviorally above; what remains is that the inline route closure inside - * `createServer` still asks them — the helper being right while the call site - * keeps writing `embeddingCheckpoint: undefined` is exactly the regression - * #2790 is about, and that closure cannot be reached without booting a server - * over a real repo + LadybugDB + embedding endpoint. Static-analysis layer of - * last resort, same precedent as api-readonly-wiring.test.ts. - */ -describe('POST /api/embed route wiring (#2790)', () => { - const readSource = () => - fs.readFile(path.join(__dirname, '..', '..', 'src', 'server', 'api.ts'), 'utf-8'); - - /** - * The body of the route's `withLbugDb` callback — everything that may only - * run while the database connection is open. Sliced rather than matched with - * a character-distance regex so a comment edit cannot silently un-assert it. - */ - const insideWithLbugDb = (source: string): string => { - // The open is a multi-line `withLbugDb(lbugPath, async () => {…}, opts)` call - // since #3091 added the FTS-mode options argument, so anchor on the call head - // and close on that options argument rather than a fixed-indent literal. - const head = source.match(/await withLbugDb\(\s*lbugPath,\s*async \(\) => \{/); - expect(head).not.toBeNull(); - const start = head!.index!; - const end = source.indexOf('skipFtsOption(ftsSession.skipFts)', start); - expect(end).toBeGreaterThan(start); - return source.slice(start, end); - }; - - it('feeds the pipeline result through resolveEmbedRunOutcome into the finalize write', async () => { - const source = await readSource(); - // The result is captured, not discarded… - expect(source).toContain('const pipelineResult = await runEmbeddingPipeline('); - // …handed to the helper with the finalize context… - expect(source).toMatch( - /resolveEmbedRunOutcome\(\s*embeddingIdentity,\s*pipelineResult,\s*finalizeContext,\s*\)/, - ); - // …and its checkpoint is what the finalize meta write persists (pre-fix: a - // hardcoded `embeddingCheckpoint: undefined`). - expect(source).toContain('embeddingCheckpoint: outcome.checkpoint'); - expect(source).toContain('partialRunError = outcome.error;'); - // A partial run does not reach `status: 'complete'`, and carries its detail. - expect(source).toMatch( - /partialRunError === undefined[\s\S]{0,400}status: 'complete'[\s\S]{0,600}status: 'failed'/, - ); - expect(source).toContain('partial: partialRunDetail,'); - }); - - it('measures after the WAL flush, inside withLbugDb, and folds the result into the write', async () => { - const source = await readSource(); - const region = insideWithLbugDb(source); - // Inside the open connection — this is the route's only chance to stamp - // `stats.embeddings`, and the next CLI run's preserve-or-wipe decision - // hangs on it. - expect(region).toContain('const measuredEmbeddings = await countPersistedEmbeddings();'); - expect(region).toContain('await saveMeta(storagePath, embeddingMeta);'); - // Ordering, without brittle character spans: flush → measure → decide → - // write. Counting before the flush would describe rows still in the WAL. - const flushed = region.lastIndexOf('await flushWAL();'); - const measured = region.indexOf('const measuredEmbeddings = await countPersistedEmbeddings();'); - const decided = region.indexOf('const outcome = resolveEmbedRunOutcome('); - const folded = region.indexOf('embeddingMeta = withMeasuredEmbeddingCount(', measured); - expect(flushed).toBeLessThan(measured); - expect(measured).toBeLessThan(decided); - expect(decided).toBeLessThan(folded); - expect(region.slice(folded)).toContain('measuredEmbeddings,'); - }); - - it('measures in the post-flush checkpoint callback and nowhere else in the pipeline options', async () => { - const source = await readSource(); - expect(source).toMatch( - /await saveEmbeddingCheckpoint\(\s*checkpoint,\s*\[\],\s*await countPersistedEmbeddings\(\),?\s*\)/, - ); - // The window-start callback fires before any row exists — it must pass no - // count rather than restate a stale one. - expect(source).toMatch( - /onCheckpointWindowStart: async \(\{ nodeIds, \.\.\.checkpoint \}\) => \{\s*await saveEmbeddingCheckpoint\(checkpoint, nodeIds\);\s*\},/, - ); - }); - - it('resolves a found checkpoint through the shared resume decision', async () => { - const source = await readSource(); - const region = insideWithLbugDb(source); - // The route asks the SAME decider the CLI does, instead of hard-throwing on - // any identity mismatch and ignoring `attempts` — the disagreement that let - // a CLI-written `'partial'` marker wedge every later `POST /api/embed`. - expect(region).toMatch(/decideEmbeddingResume\(priorCheckpoint, embeddingIdentity\)/); - // Every action is routed: abort fails the run, abandon warns and proceeds - // with an empty pending set, resume hands the decision's ids to the pipeline. - expect(region).toContain("if (resume?.action === 'abort') throw new Error(resume.error);"); - expect(region).toMatch(/resume\?\.action === 'resume'\s*\?\s*resume\.pendingNodeIds/); - // No second copy of the gate: the route no longer authors its own message. - expect(region).not.toContain('Cannot resume embedding checkpoint:'); - }); - - it('never maps the pipeline ready phase to a phase a client can read as terminal', async () => { - const source = await readSource(); - // `ready` fires unconditionally before the route knows the outcome (#2790). - expect(source).toMatch(/p\.phase === 'ready'\s*\?\s*'finalizing'/); - expect(source).not.toMatch(/p\.phase === 'ready' \? 'complete'/); - }); -}); +// The shared phase now owns checkpoint/count publication. Its runtime contract +// is covered by analysis-embedding-phase.test.ts and analysis-embedding-flow.test.ts; +// api-fts-mode.test.ts drives this route's progress, failure and completion wiring. describe('HTTP repo catalog validation', () => { const readSource = () => diff --git a/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts b/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts index 53ead617c..2fd7e0874 100644 --- a/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts +++ b/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts @@ -20,7 +20,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-embeddings-limit.test.ts b/gitnexus/test/unit/analyze-embeddings-limit.test.ts index c7f12fe7f..ee4d1aa2c 100644 --- a/gitnexus/test/unit/analyze-embeddings-limit.test.ts +++ b/gitnexus/test/unit/analyze-embeddings-limit.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts b/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts index 68798111e..c8f12d425 100644 --- a/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts +++ b/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts @@ -62,7 +62,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ isLbugReady: isLbugReadyMock, })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError, diff --git a/gitnexus/test/unit/analyze-gitnexusrc.test.ts b/gitnexus/test/unit/analyze-gitnexusrc.test.ts index e7fe6224c..d5604be10 100644 --- a/gitnexus/test/unit/analyze-gitnexusrc.test.ts +++ b/gitnexus/test/unit/analyze-gitnexusrc.test.ts @@ -50,9 +50,11 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: vi.fn(async () => null), getStoragePaths: vi.fn((repoPath: string) => ({ storagePath: path.join(repoPath, '.gitnexus'), lbugPath: path.join(repoPath, '.gitnexus', 'lbug'), + metaPath: path.join(repoPath, '.gitnexus', 'gitnexus.json'), })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts index 150cbb554..9017e0593 100644 --- a/gitnexus/test/unit/analyze-heap-respawn.test.ts +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -3,6 +3,11 @@ import { EventEmitter } from 'node:events'; const spawnMock = vi.fn(); const getHeapStatisticsMock = vi.fn(); +const npmWarningMock = vi.hoisted(() => vi.fn()); + +vi.mock('../../src/cli/resolve-invocation.js', () => ({ + warnIfNpm11NpxRisk: npmWarningMock, +})); vi.mock('child_process', async () => { const actual = await vi.importActual('child_process'); @@ -105,7 +110,37 @@ describe('analyzeCommand heap respawn', () => { else process.env.NODE_OPTIONS = initialNodeOptions; }); - it('re-execs analyze with the auto-sized heap cap (16GB-clamped) + larger semi-space and bridges progress redraw when parent is a TTY', async () => { + it('defers npm subprocess probes until the admitted working process', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + npmWarningMock.mockClear(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(npmWarningMock).not.toHaveBeenCalled(); + }); + + it('leaves the analysis pipeline unloaded in the heap supervisor', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + const load = vi.fn(() => ({ + runFullAnalysis: vi.fn(), + continuePreparedAnalysis: vi.fn(), + })); + vi.doMock('../../src/core/run-analyze.js', load); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(load).not.toHaveBeenCalled(); + } finally { + vi.doUnmock('../../src/core/run-analyze.js'); + } + }); + + it('re-execs with the admitted main heap and bridges progress redraw when parent is a TTY', async () => { delete process.env.NODE_OPTIONS; restoreStderrIsTTY = setStreamIsTTY(process.stderr, true); getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); @@ -116,9 +151,10 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=13107'); + expect(args).toContain('--max-old-space-size=3916'); expect(args).toContain('--max-semi-space-size=128'); - expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=13107'); + expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=3916'); + expect(opts.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB).toBe('4300'); expect(opts.env.NODE_OPTIONS).toContain('--max-semi-space-size=128'); expect(opts.env.GITNEXUS_RESPAWN_PROGRESS_TTY).toBe('1'); }); @@ -161,14 +197,14 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=13107'); + expect(args).toContain('--max-old-space-size=3916'); // The auto flag is appended after the ambient value, so V8's - // later-flag-wins semantics resolve to the larger cap. - expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=13107')).toBeGreaterThan( + // later-flag-wins semantics resolve to the admitted whole-heap cap. + expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=3916')).toBeGreaterThan( opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=4096'), ); const warn = cap.records().find((r) => r.msg.includes('pins the heap to 4096MB')); - expect(warn?.msg).toContain('Re-running analyze with the larger auto-sized cap'); + expect(warn?.msg).toContain('Re-running analyze with the admitted 4300MB main heap'); }); it('honors GITNEXUS_MEMORY=off: keeps the small ambient heap, silently (#2649)', async () => { @@ -229,6 +265,41 @@ describe('analyzeCommand heap respawn', () => { expect(forwardedSignalExitCode('SIGABRT', false)).toBe(1); }); + it.each(['analysis-ready', 'analysis-watch-ready'])( + 'resends startup cancellation on %s and waits for child exit', + async (ready) => { + delete process.env.NODE_OPTIONS; + const child = Object.assign(new EventEmitter(), { + connected: true, + stdout: new EventEmitter(), + stderr: new EventEmitter(), + send: vi.fn((_message, callback) => callback(null)), + }); + spawnMock.mockReturnValueOnce(child); + const { ensureHeap } = await import('../../src/cli/analyze.js'); + let settled = false; + const pending = ensureHeap({ memoryBudget: '2048' }).then(() => { + settled = true; + }); + try { + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledOnce()); + process.emit('SIGTERM'); + expect(child.send).toHaveBeenCalledExactlyOnceWith( + { type: 'cancel' }, + expect.any(Function), + ); + child.emit('message', { type: ready }); + await Promise.resolve(); + expect(child.send).toHaveBeenCalledTimes(2); + expect(settled).toBe(false); + } finally { + child.emit('close', 0, null); + await pending; + } + expect(process.exitCode).toBe(143); + }, + ); + it('GITNEXUS_MEMORY=off also disables the default (unpinned) respawn (#2649 review)', async () => { delete process.env.NODE_OPTIONS; process.env.GITNEXUS_MEMORY = 'off'; @@ -298,9 +369,9 @@ describe('analyzeCommand heap respawn', () => { .find((r) => r.msg.includes('Analysis likely ran out of memory')); expect(oomGuidance).toBeDefined(); const msg = oomGuidance?.msg ?? ''; - expect(msg).toContain('auto-sized to 13107MB'); - expect(msg).toContain('NODE_OPTIONS="--max-old-space-size="'); - expect(msg).toContain('[your-args]'); + expect(msg).toContain('admitted main heap'); + expect(msg).toContain('--resource-budget'); + expect(msg).toContain('reduce the analysis scope'); expect(msg).toContain('native crash unrelated to heap size'); cap.restore(); }); @@ -704,7 +775,7 @@ describe('ensureHeap --memory-budget (#3137)', () => { expect({ args, source: opts.env.GITNEXUS_HEAP_LIMIT_SOURCE, records: cap.records() }).toEqual({ args: [ ...preserved, - '--max-old-space-size=13107', + '--max-old-space-size=3916', '--max-semi-space-size=128', '--stack-size=4096', ...process.argv.slice(1), diff --git a/gitnexus/test/unit/analyze-http-endpoint-error.test.ts b/gitnexus/test/unit/analyze-http-endpoint-error.test.ts index 718a9fa16..71e7836f4 100644 --- a/gitnexus/test/unit/analyze-http-endpoint-error.test.ts +++ b/gitnexus/test/unit/analyze-http-endpoint-error.test.ts @@ -58,7 +58,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-job.test.ts b/gitnexus/test/unit/analyze-job.test.ts index 382a2e078..77024f04f 100644 --- a/gitnexus/test/unit/analyze-job.test.ts +++ b/gitnexus/test/unit/analyze-job.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; import { JobManager, isTerminalJobStatus, @@ -31,6 +32,38 @@ describe('JobManager', () => { expect(retrieved).toEqual(created); }); + it('keeps the original deadline when a graph worker hands off to an embedding worker', () => { + manager.dispose(); + vi.useFakeTimers(); + manager = new JobManager(); + const job = manager.createJob({ repoPath: '/tmp/repo' }); + manager.updateJob(job.id, { status: 'analyzing' }); + const child = () => + Object.assign(new EventEmitter(), { + connected: true, + exitCode: null, + signalCode: null, + send: vi.fn(), + unref: vi.fn(), + channel: { unref: vi.fn() }, + }); + const graph = child(); + manager.registerChild(job.id, graph as any); + vi.advanceTimersByTime(20 * 60_000); + graph.emit('exit', 0); + const embeddings = child(); + manager.registerChild(job.id, embeddings as any); + vi.advanceTimersByTime(10 * 60_000 - 1); + expect(embeddings.send).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(embeddings.send).toHaveBeenCalledWith({ type: 'cancel' }, expect.any(Function)); + expect(manager.hasPendingCancel(job.id)).toBe(true); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); + embeddings.emit('exit', 0); + expect(manager.getJob(job.id)?.error).toContain('30 minute limit'); + expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); + }); + it('returns undefined for unknown job id', () => { expect(manager.getJob('nonexistent')).toBeUndefined(); }); @@ -371,12 +404,12 @@ describe('JobManager', () => { expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); }); - it('cancelJob falls back to a signal when the IPC channel is already closed', () => { + it('cancelJob leaves a disconnected native worker running until it exits', () => { const job = manager.createJob({ repoPath: '/tmp/repo' }); manager.updateJob(job.id, { status: 'analyzing' }); const signals: string[] = []; - // connected:false — requestChildShutdown skips send() and signal-kills. + // The worker's disconnect handler requests cancellation at a safe point. const fakeChild = { connected: false, exitCode: null, @@ -391,10 +424,11 @@ describe('JobManager', () => { manager.cancelJob(job.id); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('cancelJob SIGKILLs after the grace period when the worker ignores IPC', () => { + it('cancelJob abandons its process reference after grace, retaining ownership until exit', () => { manager.dispose(); vi.useFakeTimers(); manager = new JobManager(); @@ -407,6 +441,8 @@ describe('JobManager', () => { exitCode: null, signalCode: null, send: () => true, + unref: vi.fn(), + channel: { unref: vi.fn() }, kill: (signal?: string) => { signals.push(signal ?? 'SIGTERM'); return true; @@ -418,10 +454,13 @@ describe('JobManager', () => { expect(manager.cancelJob(job.id)).toBe(true); expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); - expect(signals).toEqual(['SIGKILL']); + expect(signals).toEqual([]); + expect(fakeChild.unref).toHaveBeenCalledOnce(); + expect(fakeChild.channel.unref).toHaveBeenCalledOnce(); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('dispose on Windows skips immediate SIGTERM and leaves the IPC grace timer', () => { + it('dispose on Windows never force-kills a native worker', () => { manager.dispose(); vi.useFakeTimers(); vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); @@ -446,11 +485,11 @@ describe('JobManager', () => { expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); - expect(signals).toEqual(['SIGKILL']); + expect(signals).toEqual([]); vi.restoreAllMocks(); }); - it('dispose on Unix SIGTERMs after IPC and clears the grace timer', () => { + it('dispose on Unix also waits for the cooperative native safe point', () => { manager.dispose(); vi.useFakeTimers(); vi.spyOn(process, 'platform', 'get').mockReturnValue('linux'); @@ -473,9 +512,9 @@ describe('JobManager', () => { manager.registerChild(job.id, fakeChild as any); manager.dispose(); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); vi.restoreAllMocks(); }); diff --git a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts index 0bd1cff0e..1e062de63 100644 --- a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts +++ b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts @@ -36,6 +36,7 @@ const H = vi.hoisted(() => ({ // Set per-test: the only directory the fake filesystem reports as freshly // written. Anything else looks stale, exactly like a slot this job skipped. settledDir: '', + graphFile: 'lbug', })); const { forkMock, REPO_PATH } = H; @@ -44,6 +45,21 @@ vi.mock('child_process', async () => { return { ...actual, fork: H.forkMock }; }); +vi.mock('../../src/core/analysis-host-resources.js', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + return { + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: () => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4096 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, + }), + }; +}); + vi.mock('../../src/storage/repo-manager.js', () => ({ INDEX_METADATA_FILE: H.METADATA_FILE, })); @@ -56,6 +72,11 @@ vi.mock('../../src/storage/storage-resolver.js', () => ({ requireStoragePath: async () => H.STORAGE_PATH, })); +vi.mock('../../src/storage/shared-store.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, resolveGraphPath: (slot: string) => `${slot}/${H.graphFile}` }; +}); + vi.mock('node:fs', async () => { const actual = await vi.importActual('node:fs'); return { @@ -67,6 +88,8 @@ vi.mock('node:fs', async () => { const file = String(p); const dir = file.slice(0, Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\'))); if (H.settledDir && dir === H.settledDir) { + if (!file.endsWith(H.METADATA_FILE) && !file.endsWith(H.graphFile)) + throw Object.assign(new Error('Retired graph name'), { code: 'ENOENT' }); return { mtimeMs: Number.MAX_SAFE_INTEGER }; } return { mtimeMs: 0 }; @@ -135,6 +158,7 @@ describe('finalization gate follows the placement the run chose', () => { backendInit = vi.fn(async () => true); closeDbHandle = vi.fn(async () => {}); H.settledDir = ''; + H.graphFile = 'lbug'; }); afterEach(() => { @@ -200,7 +224,7 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, { branch: BRANCH }); child.emit('message', completeMessage(false)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(jobManager.getJob(job.id)?.error).toBeUndefined(); expect(backendInit).toHaveBeenCalledTimes(1); @@ -214,11 +238,43 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(backendInit).toHaveBeenCalledTimes(1); }); + it('settles an FTS-only repair of a published private generation', async () => { + vi.useFakeTimers(); + H.settledDir = H.STORAGE_PATH; + H.graphFile = `lbug.generation.${'a'.repeat(32)}`; + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher()(job, REPO_PATH, {}); + const message = completeMessage(true); + message.result.ftsRepairedOnly = true; + child.emit('message', message); + child.emit('exit', 0); + await vi.advanceTimersByTimeAsync(61_000); + expect(jobManager.getJob(job.id)?.status).toBe('complete'); + expect(backendInit).toHaveBeenCalledOnce(); + }); + + it('completes resumed vectors after graph visibility and actual worker exit without rewriting the graph', async () => { + vi.useFakeTimers(); + const releaseRepoLock = vi.fn(); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher({ releaseRepoLock })(job, REPO_PATH, { embeddings: true }); + child.emit('message', { type: 'graph-ready' }); + await vi.advanceTimersByTimeAsync(0); + expect(child.send).toHaveBeenCalledWith({ type: 'graph-visible' }); + child.emit('message', completeMessage(true)); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(releaseRepoLock).not.toHaveBeenCalled(); + child.emit('exit', 0); + await vi.advanceTimersByTimeAsync(61_000); + expect(jobManager.getJob(job.id)?.status).toBe('complete'); + expect(releaseRepoLock).toHaveBeenCalledOnce(); + }); + it('settles a first-pin (branch SET, isPrimaryBranch true) against the flat slot', async () => { // Fresh clone: first pin adopts the flat slot. The slug dir is stale, so a // gate that does `branch ? slugDir : flat` would spin the 60s timeout here. @@ -228,7 +284,7 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, { branch: BRANCH }); child.emit('message', completeMessage(true)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(jobManager.getJob(job.id)?.error).toBeUndefined(); expect(backendInit).toHaveBeenCalledTimes(1); @@ -290,6 +346,7 @@ describe('finalization gate follows the placement the run chose', () => { expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); // Must match FINALIZE_SETTLE_TIMEOUT_MS + one poll in analyze-launch.ts. + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(61_000); const done = jobManager.getJob(job.id); @@ -308,6 +365,7 @@ describe('finalization gate follows the placement the run chose', () => { const job = jobManager.createJob({ repoPath: REPO_PATH }); await launcher({ releaseRepoLock })(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); jobManager.cancelJob(job.id, 'Cancelled by user'); @@ -321,9 +379,6 @@ describe('finalization gate follows the placement the run chose', () => { expect(done?.error).toBe('Cancelled by user'); expect(done?.error).not.toMatch(/finalization not visible/); expect(backendInit).not.toHaveBeenCalled(); - expect(releaseRepoLock).not.toHaveBeenCalled(); - - child.emit('exit', 0); expect(releaseRepoLock).toHaveBeenCalledTimes(1); }); @@ -335,10 +390,10 @@ describe('finalization gate follows the placement the run chose', () => { const job = jobManager.createJob({ repoPath: REPO_PATH }); await launcher({ releaseRepoLock })(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); jobManager.cancelJob(job.id, 'Cancelled by user'); - child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('failed'); expect(releaseRepoLock).not.toHaveBeenCalled(); @@ -373,6 +428,7 @@ describe('finalization gate follows the placement the run chose', () => { expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); H.settledDir = H.STORAGE_PATH; + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(200); expect(jobManager.getJob(job.id)?.status).toBe('complete'); diff --git a/gitnexus/test/unit/analyze-launch-collapse.test.ts b/gitnexus/test/unit/analyze-launch-collapse.test.ts index 4172ead9a..86554da0f 100644 --- a/gitnexus/test/unit/analyze-launch-collapse.test.ts +++ b/gitnexus/test/unit/analyze-launch-collapse.test.ts @@ -46,6 +46,21 @@ vi.mock('child_process', async () => { // ownership-validated storage path. Pin the filesystem so the gate settles on // its FIRST poll — the gate itself is not under test here and its 200ms poll // would otherwise put a real timer between the worker message and the assertions. +vi.mock('../../src/core/analysis-host-resources.js', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + return { + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: () => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4096 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, + }), + }; +}); + vi.mock('../../src/storage/repo-manager.js', () => ({ INDEX_METADATA_FILE: H.METADATA_FILE, })); @@ -135,6 +150,9 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () const job = jobManager.createJob({ repoPath: REPO_PATH }); await launch(job, REPO_PATH, {}); child.emit('message', msg); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(backendInit).not.toHaveBeenCalled(); + child.emit('exit', 0); await vi.waitFor(() => expect(calls).toContain('updateJob:terminal')); return jobManager.getJob(job.id); @@ -191,6 +209,60 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () ); }); + it('publishes the graph before acknowledging paid work and waits for both exits', async () => { + child.pid = 12345; + const writer = child; + const embedding = makeChild(); + forkMock.mockReturnValueOnce(writer).mockReturnValueOnce(embedding); + let visible!: () => void; + backendInit.mockImplementationOnce( + () => + new Promise((resolve) => { + visible = resolve; + }), + ); + const releaseRepoLock = vi.fn(); + const launch = createLaunchAnalysisWorker({ + jobManager, + backend: { init: backendInit }, + acquireRepoLock: () => null, + releaseRepoLock, + closeDbHandle, + }); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launch(job, REPO_PATH, { embeddings: true }); + const handoff = { + version: 1, + publicationId: 'a'.repeat(32), + repoPath: REPO_PATH, + storagePath: H.STORAGE_PATH, + metadataDirectory: H.STORAGE_PATH, + repoName: REPO_NAME, + commit: 'b'.repeat(40), + }; + writer.emit('message', { type: 'graph-prepared', handoff, result: completeMessage().result }); + expect(forkMock).toHaveBeenCalledTimes(1); + expect(backendInit).not.toHaveBeenCalled(); + writer.emit('exit', 0); + expect(forkMock).toHaveBeenCalledTimes(2); + expect(embedding.send).toHaveBeenCalledWith( + expect.objectContaining({ type: 'continue', writerPid: writer.pid }), + ); + embedding.emit('message', { type: 'graph-ready' }); + await vi.waitFor(() => expect(backendInit).toHaveBeenCalledOnce()); + expect(embedding.send).not.toHaveBeenCalledWith({ type: 'graph-visible' }); + expect(releaseRepoLock).not.toHaveBeenCalled(); + visible(); + await vi.waitFor(() => expect(embedding.send).toHaveBeenCalledWith({ type: 'graph-visible' })); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + embedding.emit('message', completeMessage()); + expect(releaseRepoLock).not.toHaveBeenCalled(); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + embedding.emit('exit', 0); + await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); + expect(releaseRepoLock).toHaveBeenCalledOnce(); + }); + describe('route configuration', () => { const launchWith = async () => { const launch = createLaunchAnalysisWorker({ @@ -444,6 +516,7 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); // Must match FINALIZE_SETTLE_TIMEOUT_MS + one poll in analyze-launch.ts. + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(61_000); const done = jobManager.getJob(job.id); @@ -477,6 +550,7 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); H.settleOk = true; + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(200); expect(jobManager.getJob(job.id)?.status).toBe('complete'); @@ -548,6 +622,7 @@ describe('createLaunchAnalysisWorker — pending cancel', () => { H.settleOk = false; const job = await launchOne(); child.emit('message', completeMessage()); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); expect(jobManager.cancelJob(job.id, 'Cancelled by user')).toBe(true); H.settleOk = true; diff --git a/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts b/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts index 5e7f5190c..6ded0c2f4 100644 --- a/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts +++ b/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-local-embedding-error.test.ts b/gitnexus/test/unit/analyze-local-embedding-error.test.ts index 970dfd446..6aa4d30d8 100644 --- a/gitnexus/test/unit/analyze-local-embedding-error.test.ts +++ b/gitnexus/test/unit/analyze-local-embedding-error.test.ts @@ -66,7 +66,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts index 63a417b2e..347e9f351 100644 --- a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts +++ b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts @@ -42,7 +42,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-pagesize-error.test.ts b/gitnexus/test/unit/analyze-pagesize-error.test.ts index 687816f21..3e529450b 100644 --- a/gitnexus/test/unit/analyze-pagesize-error.test.ts +++ b/gitnexus/test/unit/analyze-pagesize-error.test.ts @@ -29,7 +29,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-phase-handoff.test.ts b/gitnexus/test/unit/analyze-phase-handoff.test.ts index 72622e0c6..fdc3263d6 100644 --- a/gitnexus/test/unit/analyze-phase-handoff.test.ts +++ b/gitnexus/test/unit/analyze-phase-handoff.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; import type { AnalyzeResult } from '../../src/core/run-analyze.js'; import type { CliPreparedAnalysisMessage } from '../../src/cli/analyze-phase-ipc.js'; @@ -6,6 +7,7 @@ type SendCallback = (error: Error | null) => void; const mocks = vi.hoisted(() => ({ run: vi.fn(), + spawn: vi.fn(), skills: vi.fn(async () => ({ skills: [{ name: 'community' }], outputPath: '/repo/.claude/skills', @@ -26,6 +28,10 @@ const mocks = vi.hoisted(() => ({ send: vi.fn((_message: unknown, callback: SendCallback) => callback(null)), }, })); +vi.mock('child_process', async (importOriginal) => ({ + ...(await importOriginal()), + spawn: mocks.spawn, +})); vi.mock('../../src/cli/analyze-phase-ipc.js', async (importOriginal) => { const actual = await importOriginal(); return { @@ -45,6 +51,11 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ LbugWipeError: class LbugWipeError extends Error {}, })); vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '/repo/.gitnexus', + metaPath: '/repo/.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => '/nonexistent/registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, @@ -118,6 +129,88 @@ afterEach(() => { }); describe('CLI graph phase handoff', () => { + it('announces cancellation readiness before starting graph work', async () => { + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISED', '1'); + vi.stubEnv('GITNEXUS_ANALYSIS_MANUAL_HEAP', '1'); + const sendReady = vi.spyOn(process, 'send').mockImplementation((message, callback) => { + if ((message as { type?: string }).type === 'analysis-ready') { + process.emit('message', { type: 'cancel' }, undefined); + if (typeof callback === 'function') callback(null); + } + return true; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(sendReady).toHaveBeenCalledWith({ type: 'analysis-ready' }, expect.any(Function)); + expect(mocks.run).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(130); + }); + + it.each(['SIGINT', 'SIGTERM', 'ipc'] as const)( + 'waits for active work after %s and removes cancellation listeners', + async (kind) => { + const events = ['SIGINT', 'SIGTERM', 'message', 'disconnect'] as const; + const before = events.map((event) => process.listenerCount(event)); + let finish!: (value: AnalyzeResult) => void; + mocks.run.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const pending = analyzeCommand(undefined, {}); + await vi.waitFor(() => expect(mocks.run).toHaveBeenCalledOnce()); + if (kind === 'ipc') process.emit('message', { type: 'cancel' }, undefined); + else process.emit(kind); + expect(mocks.run.mock.calls[0][2].signal.aborted).toBe(true); + expect(process.exit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + finish(result); + await pending; + expect(process.exitCode).toBe(kind === 'SIGTERM' ? 143 : 130); + expect(send).not.toHaveBeenCalled(); + expect(events.map((event) => process.listenerCount(event))).toEqual(before); + }, + ); + + it.each(['pin', 'off', 'ambient'] as const)( + 'keeps %s heap flags when graph work requires a supervisor', + async (kind) => { + vi.stubEnv('GITNEXUS_MEMORY', kind === 'off' ? 'off' : ''); + vi.stubEnv( + 'NODE_OPTIONS', + kind === 'ambient' ? '--max_old_space_size=65536 --max-semi-space-size=16' : '', + ); + const descriptor = Object.getOwnPropertyDescriptor(process, 'execArgv')!; + const flags = + kind === 'pin' ? ['--max_old_space_size', '2048', '--max-semi-space-size=16'] : []; + Object.defineProperty(process, 'execArgv', { configurable: true, value: flags }); + try { + const { AnalysisSupervisorRequired } = + await import('../../src/core/analysis-supervision.js'); + mocks.run.mockRejectedValueOnce(new AnalysisSupervisorRequired()); + mocks.spawn.mockImplementationOnce(() => { + const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + }); + queueMicrotask(() => child.emit('close', 0, null)); + return child; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(mocks.spawn).toHaveBeenCalledOnce(); + const [, args, options] = mocks.spawn.mock.calls[0]; + expect(args).toEqual([...flags, '--stack-size=4096', ...process.argv.slice(1)]); + expect(options.env.NODE_OPTIONS).toBe(process.env.NODE_OPTIONS); + expect(options.env.GITNEXUS_ANALYSIS_MANUAL_HEAP).toBe('1'); + } finally { + Object.defineProperty(process, 'execArgv', descriptor); + } + }, + ); + it('finishes live-graph skills before IPC and defers finalization, self-commit and success', async () => { const { analyzeCommand } = await import('../../src/cli/analyze.js'); const output = console.log; diff --git a/gitnexus/test/unit/analyze-process-budget.test.ts b/gitnexus/test/unit/analyze-process-budget.test.ts index c186ab3f4..3abc6bb07 100644 --- a/gitnexus/test/unit/analyze-process-budget.test.ts +++ b/gitnexus/test/unit/analyze-process-budget.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts b/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts index a5bfb16ee..0a861d1d8 100644 --- a/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts +++ b/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts @@ -45,7 +45,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-self-commit-bridge.test.ts b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts index d8c69dcb9..41de56c6d 100644 --- a/gitnexus/test/unit/analyze-self-commit-bridge.test.ts +++ b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts @@ -56,7 +56,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts b/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts index 99aa17547..2a3a592f9 100644 --- a/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts +++ b/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts @@ -38,7 +38,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-wal-error.test.ts b/gitnexus/test/unit/analyze-wal-error.test.ts index 097175aa2..7a73fbe6c 100644 --- a/gitnexus/test/unit/analyze-wal-error.test.ts +++ b/gitnexus/test/unit/analyze-wal-error.test.ts @@ -30,7 +30,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-wipe-error.test.ts b/gitnexus/test/unit/analyze-wipe-error.test.ts index d6d881ffa..7f405d120 100644 --- a/gitnexus/test/unit/analyze-wipe-error.test.ts +++ b/gitnexus/test/unit/analyze-wipe-error.test.ts @@ -48,7 +48,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-worker-pool-size.test.ts b/gitnexus/test/unit/analyze-worker-pool-size.test.ts index dcfc17fb6..b0f07655d 100644 --- a/gitnexus/test/unit/analyze-worker-pool-size.test.ts +++ b/gitnexus/test/unit/analyze-worker-pool-size.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-worker-timeout.test.ts b/gitnexus/test/unit/analyze-worker-timeout.test.ts index 8a2dc04ef..5b1c582c5 100644 --- a/gitnexus/test/unit/analyze-worker-timeout.test.ts +++ b/gitnexus/test/unit/analyze-worker-timeout.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index c7a986e7f..23f4bcbd0 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -13,9 +13,19 @@ const mocks = vi.hoisted(() => ({ releaseIndexLock: vi.fn(), ensurePrivateSharedGraph: vi.fn(), runEmbeddingPipeline: vi.fn(), + semanticSearch: vi.fn(async () => [ + { nodeId: 'Function:a.ts:alpha', name: 'alpha', distance: 0 }, + ]), + embedQuery: vi.fn(async () => [1, 0, 0, 0]), + runAnalysisEmbeddingPhase: vi.fn(), withLbugDb: vi.fn(), search: vi.fn(), updateJob: vi.fn(), + disconnect: vi.fn(async () => {}), + serverClose: vi.fn(), + activeJob: undefined as + | { id: string; repoName: string; status: string; progress: { graphPhase?: 'ready' } } + | undefined, })); vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ @@ -33,6 +43,12 @@ vi.mock('../../src/core/shared-store-analyze.js', () => ({ })); vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline: mocks.runEmbeddingPipeline, + semanticSearch: mocks.semanticSearch, +})); +vi.mock('../../src/core/embeddings/embedder.js', () => ({ isEmbedderReady: () => false })); +vi.mock('../../src/mcp/core/embedder.js', () => ({ embedQuery: mocks.embedQuery })); +vi.mock('../../src/core/analysis-embedding-phase.js', () => ({ + runAnalysisEmbeddingPhase: mocks.runAnalysisEmbeddingPhase, })); vi.mock('../../src/storage/storage-resolver.js', async (importOriginal) => ({ ...(await importOriginal()), @@ -55,6 +71,7 @@ vi.mock('../../src/mcp/local/local-backend.js', () => ({ async init() { return true; } + disconnect = mocks.disconnect; }, })); vi.mock('../../src/server/mcp-http.js', () => ({ @@ -72,18 +89,24 @@ vi.mock('../../src/server/grep-scan.js', () => ({ })); vi.mock('../../src/server/sse-progress.js', () => ({ mountSSEProgress: vi.fn() })); vi.mock('../../src/server/analyze-job.js', () => ({ - isTerminalJobStatus: vi.fn(() => true), + isTerminalJobStatus: (status: string) => status === 'complete' || status === 'failed', JobManager: class { + controllers: AbortController[] = []; createJob() { return { id: 'embed-job', status: 'queued' }; } updateJob = mocks.updateJob; - registerAbortController() {} + registerAbortController(_id: string, controller: AbortController) { + this.controllers.push(controller); + } + dispose() { + for (const controller of this.controllers) controller.abort(); + } getJob() { - return { status: 'complete' }; + return mocks.activeJob ?? { status: 'analyzing' }; } listJobs() { - return []; + return mocks.activeJob ? [mocks.activeJob] : []; } }, })); @@ -111,7 +134,7 @@ beforeAll(async () => { ) { app = this; queueMicrotask(args.at(-1)); - return new EventEmitter() as any; + return Object.assign(new EventEmitter(), { close: mocks.serverClose }) as any; }); try { await createServer(0); @@ -133,6 +156,7 @@ afterAll(() => { beforeEach(() => { vi.clearAllMocks(); + mocks.activeJob = undefined; mocks.acquireIndexLock.mockResolvedValue({ release: mocks.releaseIndexLock, record: { @@ -146,6 +170,7 @@ beforeEach(() => { }, }); mocks.ensurePrivateSharedGraph.mockResolvedValue(true); + mocks.runAnalysisEmbeddingPhase.mockResolvedValue({ stats: { embeddings: 2 } }); mocks.listRegisteredRepos.mockResolvedValue([entry]); mocks.withLbugDb.mockImplementation(async (_path, callback) => callback()); mocks.search.mockImplementation(async (_query, _limit, _exec, reason) => ({ @@ -154,6 +179,37 @@ beforeEach(() => { })); }); +it.each(['analyzing', 'failed'])( + 'releases a waiting graph query after publication while embedding is %s', + async (status) => { + vi.useFakeTimers(); + mocks.activeJob = { + id: 'analysis-job', + repoName: entry.name, + status: 'analyzing', + progress: {}, + }; + mocks.listRegisteredRepos.mockResolvedValueOnce([]).mockResolvedValue([entry]); + mocks.loadMeta.mockResolvedValue({ capabilities: { fts: { status: 'available' } } }); + const pending = invoke('/api/search', { repo: entry.name }); + try { + await vi.advanceTimersByTimeAsync(0); + expect(mocks.search).not.toHaveBeenCalled(); + mocks.activeJob.progress.graphPhase = 'ready'; + mocks.activeJob.status = status; + await vi.advanceTimersByTimeAsync(1000); + expect(mocks.search).toHaveBeenCalledOnce(); + await pending; + expect(mocks.updateJob).not.toHaveBeenCalled(); + } finally { + mocks.activeJob.status = 'complete'; + await vi.advanceTimersByTimeAsync(1000); + await pending.catch(() => {}); + vi.useRealTimers(); + } + }, +); + describe('POST /api/embed staged recovery preflight', () => { it('passes indexed source identity from the checkout and reports source failures', async () => { const fileHashes = { 'sample.ts': 'a'.repeat(64) }; @@ -312,18 +368,23 @@ describe('POST /api/embed staged recovery preflight', () => { expect.objectContaining({ status: 'complete' }), ); expect(mocks.ensurePrivateSharedGraph).toHaveBeenCalledTimes(1); - expect(mocks.withLbugDb).toHaveBeenCalledTimes(1); + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledTimes(1); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); expect(fs.existsSync(path.join(entry.storagePath, 'analyze.lock'))).toBe(false); }); }); -async function invoke(route: string, query: Record = {}) { +async function invoke( + route: string, + query: Record = {}, + body: Record = {}, +) { const layer = app.router.stack.find((item: any) => item.route?.path === route); expect(layer, route).toBeDefined(); const handler = layer.route.stack.at(-1).handle; const req = Object.assign(new EventEmitter(), { query, - body: { cypher: 'RETURN 1 AS value', query: 'handler', mode: 'bm25', enrich: false }, + body: { cypher: 'RETURN 1 AS value', query: 'handler', mode: 'bm25', enrich: false, ...body }, }); const res = Object.assign(new EventEmitter(), { statusCode: 200, @@ -433,8 +494,8 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => }); it.each(cases)( - 'preserves write mode while honoring $name metadata for embed', - async ({ fts, skip }) => { + 'keeps the $name graph out of the legacy writable DB path during embed', + async ({ fts }) => { mocks.loadMeta.mockResolvedValue({ capabilities: { fts } }); // This test stops at the DB boundary; it must not generate vectors or write an index. mocks.withLbugDb.mockResolvedValue(undefined); @@ -445,18 +506,82 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => expect.objectContaining({ status: 'complete' }), ), ); - expect(mocks.withLbugDb).toHaveBeenCalledExactlyOnceWith( - path.join(entry.storagePath, 'lbug'), - expect.any(Function), - skip ? { skipFts: true } : {}, + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledWith( + expect.objectContaining({ + metadataDirectory: entry.storagePath, + graphPath: path.join(entry.storagePath, 'lbug'), + }), ); - expect(mocks.loadMeta).toHaveBeenCalledTimes(2); - expect(mocks.loadMeta).toHaveBeenNthCalledWith(1, entry.storagePath); - expect(mocks.loadMeta).toHaveBeenNthCalledWith(2, entry.storagePath); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); + expect(mocks.saveMeta).not.toHaveBeenCalled(); }, ); }); +describe('POST /api/embed published-phase outcomes', () => { + it('keeps graph readiness nonterminal until the shared phase completes', async () => { + mocks.loadMeta.mockResolvedValue({ repoPath: entry.path }); + let finish!: () => void; + mocks.runAnalysisEmbeddingPhase.mockImplementation(async ({ callbacks }) => { + callbacks.onProgress('graph-ready', 90, 'Graph available', { graphPhase: 'ready' }); + await new Promise((resolve) => { + finish = resolve; + }); + return { stats: { embeddings: 2 } }; + }); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ + progress: expect.objectContaining({ graphPhase: 'ready', phase: 'graph-ready' }), + }), + ); + expect(mocks.updateJob).not.toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.releaseIndexLock).not.toHaveBeenCalled(); + finish(); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + }); + + it('surfaces incomplete vectors without clearing durable work or reporting complete', async () => { + mocks.loadMeta.mockResolvedValue({ + repoPath: entry.path, + stats: { embeddings: 1 }, + embeddingJob: { identity: 'retained' }, + }); + mocks.runAnalysisEmbeddingPhase.mockImplementation(async ({ callbacks }) => { + callbacks.onProgress('embeddings', 98, 'Incomplete', { + graphPhase: 'ready', + embeddingProgress: { phase: 'failed', completedNodes: 1, pendingNodes: 2 }, + }); + throw new Error('Provider unavailable; resume durable progress'); + }); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ + status: 'failed', + error: expect.stringContaining('resume durable progress'), + }), + ); + expect(mocks.updateJob).not.toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); + }); +}); + describe('GET /api/search FTS warning redaction', () => { afterEach(() => { resetExtensionState(); @@ -490,3 +615,59 @@ describe('GET /api/repos catalog validation', () => { expect(mocks.listRegisteredRepos).toHaveBeenCalledTimes(1); }); }); + +it.each(['semantic', 'hybrid'])( + 'searches published vectors in %s mode without requiring a previously loaded model', + async (mode) => { + const metadata = { + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + }; + mocks.loadMeta.mockResolvedValue(metadata); + await invoke('/api/search', {}, { mode }); + expect(mocks.semanticSearch).toHaveBeenCalledOnce(); + const args = mocks.semanticSearch.mock.calls[0] as unknown[]; + expect(args[4]).toEqual({ + metadataDirectory: entry.storagePath, + metadata, + graphPath: path.join(entry.storagePath, 'lbug'), + }); + expect(args[5]).toBeTypeOf('function'); + // Only the verified vector reader invokes this lazy callback. + expect(mocks.embedQuery).not.toHaveBeenCalled(); + }, +); + +it('waits for embedding native cleanup before closing the backend or exiting on shutdown', async () => { + mocks.loadMeta.mockResolvedValue({ repoPath: entry.path }); + let finish!: () => void; + let signal!: AbortSignal; + mocks.runAnalysisEmbeddingPhase.mockImplementation(async (options) => { + signal = options.signal; + await new Promise((resolve) => { + finish = resolve; + }); + signal.throwIfAborted(); + }); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledOnce()); + const shutdown = process + .listeners('SIGTERM') + .find((listener) => !originalListeners.get('SIGTERM')!.includes(listener)); + expect(shutdown).toBeDefined(); + const stopped = (shutdown as unknown as () => Promise)(); + try { + await new Promise((resolve) => setImmediate(resolve)); + expect(signal.aborted).toBe(true); + expect(mocks.serverClose).toHaveBeenCalledOnce(); + expect(mocks.releaseIndexLock).not.toHaveBeenCalled(); + expect(mocks.disconnect).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + } finally { + finish(); + await stopped; + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + exit.mockRestore(); + } + expect(mocks.disconnect).toHaveBeenCalledOnce(); +}); diff --git a/gitnexus/test/unit/api-readonly-wiring.test.ts b/gitnexus/test/unit/api-readonly-wiring.test.ts index e53489966..32b7fff0f 100644 --- a/gitnexus/test/unit/api-readonly-wiring.test.ts +++ b/gitnexus/test/unit/api-readonly-wiring.test.ts @@ -67,29 +67,28 @@ describe('api read-only endpoint wiring', () => { ); }); - it('/api/embed refuses an in-place FTS crash WAL before the writable open', async () => { + it('/api/embed refuses an in-place FTS crash WAL before the vector phase', async () => { const source = await readSource(); const embedSection = source.match( - /\/\/ Run embedding pipeline asynchronously[\s\S]*?skipFtsOption\(ftsSession\.skipFts\)/, + /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, ); expect(embedSection).not.toBeNull(); const gateIdx = embedSection![0].indexOf('assertReadOnlyFtsCrashSafe(lbugPath)'); - const openIdx = embedSection![0].indexOf('await withLbugDb('); + const openIdx = embedSection![0].indexOf('await runAnalysisEmbeddingPhase('); expect(gateIdx).toBeGreaterThan(-1); expect(openIdx).toBeGreaterThan(gateIdx); expect(embedSection![0]).not.toMatch(/fts-inplace-checkpointed/); expect(embedSection![0]).not.toMatch(/readOnly:\s*true/); }); - it('/api/embed remains write-mode (writes embeddings — must not be flipped to readOnly)', async () => { + it('/api/embed delegates vector writes without reopening the graph in legacy write mode', async () => { const source = await readSource(); - // Negative assertion: no `readOnly: true` between the embed job's - // `runEmbeddingPipeline` call site and its withLbugDb open. Embed writes - // back vector rows; flipping this to readOnly would silently break it. - const embedSection = source.match(/runEmbeddingPipeline[\s\S]{0,400}\}\s*\)\s*;[\s\S]{0,200}/); - if (embedSection) { - expect(embedSection[0]).not.toMatch(/readOnly:\s*true/); - } + const embedSection = source.match( + /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, + ); + expect(embedSection).not.toBeNull(); + expect(embedSection![0]).toContain('await runAnalysisEmbeddingPhase('); + expect(embedSection![0]).not.toContain('await withLbugDb('); }); it('/api/embed keeps the repository lock until cancelled work actually stops', async () => { @@ -107,9 +106,9 @@ describe('api read-only endpoint wiring', () => { /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, ); expect(embedSection).not.toBeNull(); - expect(embedSection?.[0]).toContain('forceReembedNodeIds'); - expect(embedSection?.[0]).toContain('onCheckpointWindowStart'); - expect(embedSection?.[0]).toContain('pendingNodeIds'); - expect(embedSection?.[0]).toContain('saveMeta'); + expect(embedSection?.[0]).toContain('await runAnalysisEmbeddingPhase('); + expect(embedSection?.[0]).toContain('metadataDirectory: storagePath'); + expect(embedSection?.[0]).toContain('signal: embedController.signal'); + expect(embedSection?.[0]).not.toContain('embeddingCheckpoint: undefined'); }); }); diff --git a/gitnexus/test/unit/augmentation-generation.test.ts b/gitnexus/test/unit/augmentation-generation.test.ts new file mode 100644 index 000000000..d56bc1410 --- /dev/null +++ b/gitnexus/test/unit/augmentation-generation.test.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { branchSlug } from '../../src/storage/branch-index.js'; + +const mocks = vi.hoisted(() => ({ repos: vi.fn(), branch: vi.fn(), init: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', () => ({ listRegisteredRepos: mocks.repos })); +vi.mock('../../src/storage/storage-resolver.js', () => ({ + STATUS_STORAGE_REQUIREMENTS: {}, + requireRegisteredStoragePath: async (entry: { storagePath: string }) => entry.storagePath, +})); +vi.mock('../../src/storage/git.js', async (importOriginal) => ({ + ...(await importOriginal()), + getCurrentBranch: mocks.branch, +})); +vi.mock('../../src/core/lbug/pool-adapter.js', () => ({ + initLbug: mocks.init, + isLbugReady: () => false, + executeQuery: async () => [], +})); +vi.mock('../../src/core/search/bm25-index.js', () => ({ + searchFTSFromLbug: async () => ({ results: [], ftsAvailable: true }), +})); +import { augment } from '../../src/core/augmentation/engine.js'; + +let root: string; +beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-augment-generation-')); +}); +afterEach(async () => fs.rm(root, { recursive: true, force: true })); + +it.each([undefined, 'feature/generation'])( + 'opens the published graph in slot %s', + async (branch) => { + const storagePath = path.join(root, '.gitnexus'); + const slot = branch ? path.join(storagePath, 'branches', branchSlug(branch)) : storagePath; + const graphPath = `lbug.generation.${'a'.repeat(32)}`; + await fs.mkdir(slot, { recursive: true }); + await fs.writeFile(path.join(slot, 'gitnexus.json'), JSON.stringify({ graphPath })); + await fs.writeFile(path.join(slot, graphPath), 'owned graph fixture'); + mocks.branch.mockReturnValue(branch ?? 'main'); + mocks.repos.mockResolvedValue([ + { name: 'fixture', path: root, storagePath, branches: branch ? [{ branch }] : [] }, + ]); + + await augment('login', root); + + expect(mocks.init).toHaveBeenCalledWith('fixture', path.join(slot, graphPath)); + }, +); diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 32ad10fa7..d57bc6ec2 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -1,18 +1,43 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, vi } from 'vitest'; -const { autoHeapCapMbMock } = vi.hoisted(() => ({ autoHeapCapMbMock: vi.fn(() => 512) })); -vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ - autoHeapCapMb: autoHeapCapMbMock, -})); - import { AutoSyncAnalysisError, - createAutoSyncAnalysisRunner, + createAutoSyncAnalysisRunner as createRunner, + type AutoSyncAnalysisRunner, } from '../../src/core/auto-sync/analysis-worker-launch.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; + +function createAutoSyncAnalysisRunner(overrides: Parameters[0]) { + const run = createRunner(overrides); + return (...args: Parameters) => { + const slots = args[5] ?? 1; + args[1] = { + ...args[1], + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: (16 * 1024 ** 3) / slots, + availableCpus: Math.max(1, 8 / slots), + mainHeapMb: 2048 / slots, + modelMemoryMb: 0, + }), + }; + return run(...args); + }; +} + +const handoff = { + version: 1 as const, + publicationId: 'a'.repeat(32), + repoPath: '/tmp/repo', + storagePath: '/tmp/repo/.gitnexus', + metadataDirectory: '/tmp/repo/.gitnexus', + repoName: 'repo', + commit: 'b'.repeat(40), +}; function createChild() { return Object.assign(new EventEmitter(), { + pid: 1234, send: vi.fn(), stdout: { resume: vi.fn() }, stderr: { resume: vi.fn() }, @@ -20,6 +45,55 @@ function createChild() { } describe('auto-sync analysis worker', () => { + it('continues after actual exit and retains the original timeout across phases', async () => { + const graph = createChild(); + const embedding = createChild(); + const forkWorker = vi.fn().mockReturnValueOnce(graph).mockReturnValueOnce(embedding); + const clearTimeoutFn = vi.fn(); + const run = createAutoSyncAnalysisRunner({ + forkWorker, + setTimeoutFn: vi.fn(() => 1) as any, + clearTimeoutFn: clearTimeoutFn as any, + }); + const pending = run('/tmp/repo', {}, 50); + const prepared = { type: 'graph-prepared', handoff, result: { stats: { files: 3 } } }; + graph.emit('message', prepared); + expect(forkWorker).toHaveBeenCalledTimes(1); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + graph.emit('exit', 0, null); + expect(forkWorker).toHaveBeenCalledTimes(2); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + expect(embedding.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'continue', + prepared, + writerPid: graph.pid, + }), + ); + embedding.emit('message', { type: 'graph-ready' }); + expect(embedding.send).toHaveBeenLastCalledWith({ type: 'graph-visible' }); + let done = false; + void pending.then(() => { + done = true; + }); + embedding.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + await Promise.resolve(); + expect(done).toBe(false); + embedding.emit('exit', 0, null); + await expect(pending).resolves.toEqual({ stats: { files: 3 } }); + }); + + it('never starts a continuation after an unsuccessful graph exit', async () => { + const child = createChild(); + const forkWorker = vi.fn(() => child as any); + const run = createAutoSyncAnalysisRunner({ forkWorker }); + const pending = run('/tmp/repo', {}, 50); + child.emit('message', { type: 'graph-prepared', handoff, result: { stats: {} } }); + child.emit('exit', 1, null); + await expect(pending).rejects.toThrow('exited before completion'); + expect(forkWorker).toHaveBeenCalledTimes(1); + }); + it('does not treat graph-prepared as terminal or clear the active timeout', async () => { const child = createChild(); const clearTimeoutFn = vi.fn(); @@ -29,7 +103,7 @@ describe('auto-sync analysis worker', () => { clearTimeoutFn: clearTimeoutFn as any, }); const result = run('/tmp/repo', {}, 50); - child.emit('message', { type: 'graph-prepared', handoff: {}, result: { stats: { files: 3 } } }); + child.emit('message', { type: 'graph-prepared', handoff, result: { stats: { files: 3 } } }); expect(clearTimeoutFn).not.toHaveBeenCalled(); child.emit('message', { type: 'error', message: 'cancelled after preparing' }); child.emit('exit', 0, null); @@ -44,7 +118,7 @@ describe('auto-sync analysis worker', () => { const result = run('/tmp/repo', { branch: 'main' }, 50); expect(forkWorker).toHaveBeenCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=512']), + expect.arrayContaining(['--max-old-space-size=2000']), ); child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); @@ -249,13 +323,13 @@ describe('auto-sync analysis worker', () => { const parallel = run('/tmp/repo', { branch: 'main' }, 50, undefined, undefined, 4); expect(forkWorker).toHaveBeenLastCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=128']), + expect.arrayContaining(['--max-old-space-size=464']), ); const solo = run('/tmp/repo', { branch: 'main' }, 50); expect(forkWorker).toHaveBeenLastCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=512']), + expect.arrayContaining(['--max-old-space-size=2000']), ); for (const child of forkChildren) { diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 92461bbe2..6da1df66f 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -22,6 +22,7 @@ const { executeWithReusedStatementMock, fetchExistingEmbeddingHashesMock, runEmbeddingPipelineMock, + runAnalysisEmbeddingPhaseMock, resolveEmbeddingIdentityMock, installEmbeddingRuntimeMock, resolveEmbeddingRuntimeMock, @@ -40,6 +41,7 @@ const { executeWithReusedStatementMock: vi.fn(), fetchExistingEmbeddingHashesMock: vi.fn(), runEmbeddingPipelineMock: vi.fn(), + runAnalysisEmbeddingPhaseMock: vi.fn(), resolveEmbeddingIdentityMock: vi.fn(), installEmbeddingRuntimeMock: vi.fn(), resolveEmbeddingRuntimeMock: vi.fn(), @@ -74,6 +76,10 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ fetchExistingEmbeddingHashes: (...args: unknown[]) => fetchExistingEmbeddingHashesMock(...args), })); +vi.mock('../../src/core/analysis-embedding-phase.js', () => ({ + runAnalysisEmbeddingPhase: (...args: unknown[]) => runAnalysisEmbeddingPhaseMock(...args), +})); + vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline: (...args: unknown[]) => runEmbeddingPipelineMock(...args), })); @@ -164,6 +170,9 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { loadMetaMock.mockReset().mockResolvedValue({ ...BASE_META }); saveMetaMock.mockReset().mockResolvedValue(undefined); initLbugMock.mockReset().mockResolvedValue(undefined); + runAnalysisEmbeddingPhaseMock + .mockReset() + .mockResolvedValue({ ...BASE_META, stats: { embeddings: 2 } }); closeLbugMock.mockReset().mockResolvedValue(undefined); executeQueryMock.mockReset().mockResolvedValue([{ cnt: 2 }]); executeWithReusedStatementMock.mockReset(); @@ -205,8 +214,9 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { order.push('loadMeta'); return { ...BASE_META }; }); - initLbugMock.mockImplementation(async () => { - order.push('init'); + runAnalysisEmbeddingPhaseMock.mockImplementation(async () => { + order.push('phase'); + return BASE_META; }); await run(); @@ -214,7 +224,7 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { expect(acquireIndexLockMock).toHaveBeenCalledWith(dir, { sweep: false }); expect(order[0]).toBe('lock'); expect(order.indexOf('loadMeta')).toBeGreaterThan(order.indexOf('lock')); - expect(order.indexOf('init')).toBeGreaterThan(order.indexOf('loadMeta')); + expect(order.indexOf('phase')).toBeGreaterThan(order.indexOf('loadMeta')); expect(order.at(-1)).toBe('release'); }); @@ -406,119 +416,51 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await run(); expect(ensurePrivateSharedGraphMock).toHaveBeenCalledTimes(1); - expect(runEmbeddingPipelineMock).toHaveBeenCalledTimes(1); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledTimes(1); expect(releaseMock).toHaveBeenCalledTimes(1); expect(existsSync(path.join(dir, 'analyze.lock'))).toBe(false); }); - it('supplies the checkout root and indexed source policy, not the storage directory', async () => { + it('does not clear durable partial work when the shared phase fails', async () => { const { dir } = await store(); - const fileHashes = { 'sample.ts': 'a'.repeat(64) }; - loadMetaMock.mockResolvedValue({ ...BASE_META, fileHashes, contentRetention: 'symbol' }); - await run(); - expect(runEmbeddingPipelineMock.mock.calls[0]?.[6].source).toEqual({ - repoPath: BASE_META.repoPath, - fileHashes, - contentRetention: 'symbol', - }); - expect(dir).not.toBe(BASE_META.repoPath); - }); - - it('persists an interrupted checkpoint from the pipeline checkpoint callbacks', async () => { - // The resume contract lives in these callbacks; a mock that never invokes - // them leaves the whole save path unexecuted. - await store(); - runEmbeddingPipelineMock.mockImplementation( - async ( - _executeQuery: unknown, - _executeWithReusedStatement: unknown, - _onProgress: unknown, - _config: unknown, - _signal: unknown, - _existing: unknown, - options: { - onCheckpointWindowStart: (checkpoint: Record) => Promise; - onCheckpoint: (checkpoint: Record) => Promise; - }, - ) => { - await options.onCheckpointWindowStart({ - nodeIds: ['n2', 'n3'], - nodesProcessed: 1, - totalNodes: 3, - chunksProcessed: 1, - }); - await options.onCheckpoint({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 3 }); - return { nodesProcessed: 3, chunksProcessed: 3, failedNodeIds: [] }; - }, + const retained = { ...BASE_META, embeddingJob: { identity: 'retained-work' } }; + loadMetaMock.mockResolvedValue(retained); + runAnalysisEmbeddingPhaseMock.mockRejectedValue(new Error('provider unavailable')); + await expect(run()).rejects.toThrow('provider unavailable'); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + metadataDirectory: dir, + nodeLimit: 0, + }), ); - - await run(); - - type SavedMeta = { - stats?: { embeddings?: number }; - embeddingCheckpoint?: { pendingNodeIds?: string[] }; - }; - const windowStart = saveMetaMock.mock.calls[0]?.[1] as SavedMeta; - expect(windowStart.embeddingCheckpoint?.pendingNodeIds).toEqual(['n2', 'n3']); - // The window marker carries no count, so the last known one must survive. - expect(windowStart.stats?.embeddings).toBe(1); - - const windowEnd = saveMetaMock.mock.calls[1]?.[1] as SavedMeta; - expect(windowEnd.embeddingCheckpoint?.pendingNodeIds).toEqual([]); - expect(windowEnd.stats?.embeddings).toBe(2); + expect(saveMetaMock).not.toHaveBeenCalled(); + expect(runEmbeddingPipelineMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); }); - it('keeps a partial checkpoint when some nodes failed to embed', async () => { + it('keeps the shared phase error without opening or closing a legacy writable graph', async () => { await store(); - runEmbeddingPipelineMock.mockResolvedValue({ - nodesProcessed: 2, - chunksProcessed: 2, - failedNodeIds: ['n9'], - }); - - await run(); - - const saved = saveMetaMock.mock.calls.at(-1)?.[1] as { - embeddingCheckpoint?: { pendingNodeIds?: string[] }; - }; - expect(saved.embeddingCheckpoint).toBeDefined(); - expect(saved.embeddingCheckpoint?.pendingNodeIds).toEqual(['n9']); - }); - - it('does not publish a missing count cell as zero', async () => { - await store(); - executeQueryMock.mockResolvedValue([{}]); - - await expect(run()).rejects.toThrow('Could not verify persisted embedding count.'); - expect(saveMetaMock).toHaveBeenCalledTimes(1); - const saved = saveMetaMock.mock.calls[0]?.[1] as { - stats?: { embeddings?: number }; - embeddingCheckpoint?: { kind?: string; pendingNodeIds?: string[] }; - }; - expect(saved.stats?.embeddings).toBe(1); - expect(saved.embeddingCheckpoint?.kind).toBe('unverified-count'); - expect(saved.embeddingCheckpoint?.pendingNodeIds).toEqual([]); - expect(releaseMock).toHaveBeenCalled(); - }); - - it('keeps the pipeline error when closeLbug also rejects', async () => { - await store(); - runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom')); + runAnalysisEmbeddingPhaseMock.mockRejectedValue(new Error('phase boom')); closeLbugMock.mockRejectedValue(new Error('close boom')); - - await expect(run()).rejects.toThrow('pipeline boom'); - expect(releaseMock).toHaveBeenCalled(); - }); - - it('keeps the pipeline error when sidecar reap also throws', async () => { - await store(); - runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom')); reapEmbeddingSidecarMock.mockImplementation(() => { throw new Error('reap boom'); }); + await expect(run()).rejects.toThrow('phase boom'); + expect(initLbugMock).not.toHaveBeenCalled(); + expect(closeLbugMock).not.toHaveBeenCalled(); + expect(reapEmbeddingSidecarMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); + }); - await expect(run()).rejects.toThrow('pipeline boom'); - expect(releaseMock).toHaveBeenCalled(); + it('preserves a prepared graph receipt before any embedding or staging cleanup', async () => { + const { dir } = await store(); + const receipt = path.join(dir, 'analysis-publication.json'); + await writeFile(receipt, '{ malformed retained receipt'); + await expect(run()).rejects.toThrow(); + expect(await readFile(receipt, 'utf8')).toBe('{ malformed retained receipt'); + expect(runAnalysisEmbeddingPhaseMock).not.toHaveBeenCalled(); + expect(ensurePrivateSharedGraphMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); }); it('does not spawn npm on darwin/x64', async () => { @@ -561,22 +503,29 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { it('does not statically import the embedding pipeline', async () => { const { readFileSync } = await import('node:fs'); const src = readFileSync(new URL('../../src/cli/embeddings-sync.ts', import.meta.url), 'utf8'); - expect(src).not.toMatch(/^import .*embedding-pipeline/m); - expect(src).toContain("await import('../core/embeddings/embedding-pipeline.js')"); + expect(src).not.toMatch(/^import .*analysis-embedding-phase/m); + expect(src).toContain("await import('../core/analysis-embedding-phase.js')"); }); - it('loads existing hashes without materializing cached vectors', async () => { - await store(); - const hashes = new Map([ - ['n1', 'h1'], - ['n2', 'h2'], - ]); - fetchExistingEmbeddingHashesMock.mockResolvedValue(hashes); - + it('passes the owned graph generation without materializing hashes or vectors', async () => { + const { dir, metaPath } = await store(); + const metadata = { + ...BASE_META, + graphPath: 'lbug.generation.' + 'a'.repeat(32), + graphGeneration: { id: 'a'.repeat(32) }, + }; + await writeFile(metaPath, JSON.stringify(metadata)); + await writeFile(path.join(dir, metadata.graphPath), 'owned graph'); + loadMetaMock.mockResolvedValue(metadata); await run(); - - expect(fetchExistingEmbeddingHashesMock).toHaveBeenCalledTimes(1); - const existingArg = runEmbeddingPipelineMock.mock.calls[0]?.[5]; - expect(existingArg).toBe(hashes); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + graphPath: path.join(dir, metadata.graphPath), + expectedGeneration: metadata.graphGeneration.id, + }), + ); + expect(fetchExistingEmbeddingHashesMock).not.toHaveBeenCalled(); + expect(runEmbeddingPipelineMock).not.toHaveBeenCalled(); + expect(saveMetaMock).not.toHaveBeenCalled(); }); }); diff --git a/gitnexus/test/unit/lbug-checkpoint.test.ts b/gitnexus/test/unit/lbug-checkpoint.test.ts index f898030e2..d13a385c3 100644 --- a/gitnexus/test/unit/lbug-checkpoint.test.ts +++ b/gitnexus/test/unit/lbug-checkpoint.test.ts @@ -56,7 +56,7 @@ describe('flushWAL / safeClose — consolidation guard (#1376)', () => { it('closeLbug delegates to safeClose instead of inlining conn.close/db.close', () => { // Match `closeLbug =` precisely so we don't prefix-match `closeLbugBeforeExit`. const closeLbugBody = adapterSource.slice(adapterSource.indexOf('export const closeLbug =')); - expect(closeLbugBody).toMatch(/await safeClose\(\)/); + expect(closeLbugBody).toMatch(/await safeClose\(options\)/); // closeLbug must NOT contain its own conn.close() or db.close() — those // live exclusively inside safeClose now. const closeLbugBlock = closeLbugBody.slice(0, closeLbugBody.indexOf('export const', 1) >>> 0); diff --git a/gitnexus/test/unit/lbug-resource-release.test.ts b/gitnexus/test/unit/lbug-resource-release.test.ts index c125974b6..7f01d26c2 100644 --- a/gitnexus/test/unit/lbug-resource-release.test.ts +++ b/gitnexus/test/unit/lbug-resource-release.test.ts @@ -1,6 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type lbug from '@ladybugdb/core'; import type { LbugConnectionHandle } from '../../src/core/lbug/lbug-config.js'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; type LbugModule = typeof lbug; @@ -148,15 +152,19 @@ describe('reserved native resource release', () => { const onProgress = vi.fn(); const loadMetadata = vi.fn(); const provider = vi.fn(); + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-release-admission-')); + execFileSync('git', ['init', '--quiet', repo]); + vi.stubEnv('GITNEXUS_HOME', path.join(repo, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); try { - await expect(runFullAnalysis('/unused', { resources }, { onProgress })).rejects.toThrow( + await expect(runFullAnalysis(repo, { resources }, { onProgress })).rejects.toThrow( 'previous allocation is live', ); await expect( runPublishedEmbeddingJob( { - metadataDirectory: '/unused', - graphPath: '/unused/graph', + metadataDirectory: path.join(repo, '.gitnexus'), + graphPath: path.join(repo, '.gitnexus', 'lbug'), resources: resources.embedding, metadata: { loadMetadata, saveMetadata: vi.fn() }, }, @@ -169,6 +177,8 @@ describe('reserved native resource release', () => { } finally { h.db.close.mockResolvedValue(); await native.drainLbugResourceReleases(); + vi.unstubAllEnvs(); + await fs.rm(repo, { recursive: true, force: true }); } }); }); diff --git a/gitnexus/test/unit/rate-limit.test.ts b/gitnexus/test/unit/rate-limit.test.ts index 90e428b25..545efdca7 100644 --- a/gitnexus/test/unit/rate-limit.test.ts +++ b/gitnexus/test/unit/rate-limit.test.ts @@ -313,10 +313,10 @@ describe('production routes — rate-limit middleware wiring', () => { ); }); - it('embed route flushes WAL via flushWAL, not inline executeQuery (#1376)', () => { - // The embed handler must call the consolidated helper, not hand-roll - // its own try/catch around executeQuery('CHECKPOINT'). - expect(apiSource).toMatch(/await flushWAL\(\)/); + it('embed route delegates durable publication to the shared phase (#1376)', () => { + // The phase owns the vector writer and its durable checkpoint. The HTTP + // reader must not checkpoint its read-only graph connection. + expect(apiSource).toMatch(/await runAnalysisEmbeddingPhase\(/); expect(apiSource).not.toMatch(/executeQuery\('CHECKPOINT'\)/); }); }); diff --git a/gitnexus/test/unit/shared-store-generations.test.ts b/gitnexus/test/unit/shared-store-generations.test.ts index c332d7c3f..5ab5e4acc 100644 --- a/gitnexus/test/unit/shared-store-generations.test.ts +++ b/gitnexus/test/unit/shared-store-generations.test.ts @@ -10,13 +10,32 @@ import { seedSharedSlot, } from '../../src/core/shared-store-analyze.js'; import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; -import { reclaimSharedStore } from '../../src/storage/shared-store-lifecycle.js'; +import { + describeSharedGraph, + reclaimSharedStore, +} from '../../src/storage/shared-store-lifecycle.js'; import { resolveGraphPath, resolveGraphWritePath, sharedStoreLayout, } from '../../src/storage/shared-store.js'; import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../../src/storage/analysis-slot-lock.js'; + +it('reports named slot-local generations as private and commit graphs as shared', () => { + const slot = path.resolve('store/checkouts/owner'); + expect(describeSharedGraph(path.join(slot, 'lbug'), slot)).toBe('private'); + expect(describeSharedGraph(path.join(slot, `lbug.generation.${'a'.repeat(32)}`), slot)).toBe( + 'private', + ); + expect(describeSharedGraph(path.resolve('store/commits/commit-feature/lbug'), slot)).toBe( + 'shared', + ); + expect(describeSharedGraph(path.resolve('store/checkouts/owner-other/lbug'), slot)).toBe( + 'shared', + ); +}); describe('shared-store logical generations', () => { let root: string; @@ -126,6 +145,71 @@ describe('shared-store logical generations', () => { expect(copy).not.toHaveBeenCalled(); }); + it('keeps the private graph while historical vectors still belong to its migration', async () => { + const graph = path.join(slot, generation); + await fs.writeFile(graph, 'graph with preserved legacy vectors'); + const meta = modern({ + graphPath: generation, + legacyVectorSource: 'e'.repeat(64), + embeddingDims: 4, + stats: { embeddings: 3 }, + }); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(graph); + expect(await loadMeta(slot)).toMatchObject(meta); + expect(await fs.readFile(graph, 'utf8')).toBe('graph with preserved legacy vectors'); + }); + + it('does not collect an orphan member while a branch writer owns its lock', async () => { + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + const lock = await acquireIndexLock(branch, { sweep: false }); + try { + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([]); + expect((await fs.stat(branch)).isDirectory()).toBe(true); + } finally { + lock.release(); + } + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([slot]); + }); + + it('admits new branch writers under the member gate, then releases it for sibling work', async () => { + const gate = await acquireIndexLock(slot, { sweep: false }); + const branch = path.join(slot, 'branches', 'new-branch'); + try { + await expect( + acquireAnalysisSlotLock(branch, { timeoutMs: 1, sweep: false }), + ).rejects.toThrow(); + } finally { + gate.release(); + } + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1000, sweep: false }); + try { + const rootWriter = await acquireIndexLock(slot, { timeoutMs: 1000, sweep: false }); + rootWriter.release(); + await expect(acquireIndexLock(branch, { timeoutMs: 1, sweep: false })).rejects.toThrow(); + } finally { + writer.release(); + } + }); + + it.each(['e'.repeat(64), null])( + 'retains an orphan with historical vector evidence: %s', + async (source) => { + await saveMeta( + slot, + metadata({ + repoPath: path.join(root, 'removed-checkout'), + legacyVectorSource: source as unknown as string, + }), + ); + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([]); + expect(await loadMeta(slot)).toHaveProperty('legacyVectorSource', source); + }, + ); + it('seeds independent completed vector pins that survive collection of their source', async () => { await fs.writeFile(path.join(slot, generation), 'completed graph'); const original = await writeVector(slot); diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts index 81afe3136..24f2902d0 100644 --- a/gitnexus/test/unit/vector-generation.test.ts +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -64,6 +64,24 @@ describe('artifact identity and pending-job routing', () => { }); describe('vector generation routing', () => { + it('keeps preserved monolithic vectors readable when a generation was published but embedding was capped', () => { + const preserved: RepoMeta = { + ...meta, + vectorGeneration: undefined, + legacyVectorSource: 'f'.repeat(64), + embeddingDims: 4, + stats: { embeddings: 2 }, + }; + expect(select(preserved)).toEqual({ kind: 'legacy' }); + for (const changed of [ + { legacyVectorSource: '../foreign' }, + { stats: { embeddings: 0 } }, + { embeddingDims: 0 }, + { embeddingJob: { version: 1 as const, identity, graphGeneration: generation } }, + ]) + expect(select({ ...preserved, ...changed })).toMatchObject({ kind: 'unavailable' }); + expect(select(preserved, '/slot/foreign')).toMatchObject({ kind: 'unavailable' }); + }); it.each([ { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }, { graphWriteCollapsed: { expected: 100, persisted: 1 } }, diff --git a/gitnexus/test/unit/watch-cancellation.test.ts b/gitnexus/test/unit/watch-cancellation.test.ts new file mode 100644 index 000000000..3e89dd531 --- /dev/null +++ b/gitnexus/test/unit/watch-cancellation.test.ts @@ -0,0 +1,80 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ analyze: vi.fn(), mayEmbed: vi.fn(async () => false) })); +vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: vi.fn(async () => false) })); +vi.mock('../../src/core/run-analyze.js', () => ({ + runFullAnalysis: mocks.analyze, + analyzeFailureMayHaveMutatedLiveIndex: () => false, +})); +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: mocks.mayEmbed, + resolveHostAnalysisResources: () => ({ mainHeapMb: 512 }), +})); +import { watchCommandWithRunnerIdentity } from '../../src/cli/analyze-watch.js'; +import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + mocks.analyze.mockReset(); + mocks.mayEmbed.mockReset(); + mocks.mayEmbed.mockResolvedValue(false); +}); + +it.each(['preflight', 'analysis'] as const)( + 'consumes supervisor cancellation during %s and waits for active work', + async (phase) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-cancel-')); + const repo = path.join(root, 'repo'); + await fs.mkdir(repo); + execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: repo, stdio: 'pipe' }); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.spyOn(process, 'send').mockImplementation(() => true); + const listeners = process.listeners('message'); + let finish!: () => void; + const held = new Promise((resolve) => { + finish = resolve; + }); + if (phase === 'preflight') + mocks.mayEmbed.mockImplementationOnce(async () => { + await held; + return false; + }); + mocks.analyze.mockImplementation(async () => { + await held; + return { alreadyUpToDate: true }; + }); + let settled = false; + const run = watchCommandWithRunnerIdentity( + { schemaVersion: 4 } as AnalyzerRunnerIdentity, + repo, + ).finally(() => { + settled = true; + }); + try { + await vi.waitFor(() => + expect(phase === 'preflight' ? mocks.mayEmbed : mocks.analyze).toHaveBeenCalled(), + ); + const control = process + .listeners('message') + .find((listener) => !listeners.includes(listener)); + expect(control).toBeDefined(); + control!({ type: 'cancel' }, undefined); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + finish(); + await run; + expect(process.listeners('message')).not.toContain(control); + expect(mocks.analyze).toHaveBeenCalledTimes(phase === 'preflight' ? 0 : 1); + } finally { + finish(); + await run; + await fs.rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/gitnexus/test/unit/watch-paths.test.ts b/gitnexus/test/unit/watch-paths.test.ts index 0a019d6bb..3e39deea6 100644 --- a/gitnexus/test/unit/watch-paths.test.ts +++ b/gitnexus/test/unit/watch-paths.test.ts @@ -5,6 +5,15 @@ import path from 'node:path'; import { createWatchIgnorePredicate } from '../../src/config/ignore-service.js'; import { isRelevantWatchPath, resolveWatchOptions } from '../../src/cli/analyze-watch.js'; import * as git from '../../src/storage/git.js'; +import { + analysisMayEmbed, + resolveHostAnalysisResources, +} from '../../src/core/analysis-host-resources.js'; + +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: vi.fn(async () => false), + resolveHostAnalysisResources: vi.fn(() => ({ mainHeapMb: 512 })), +})); vi.mock('../../src/storage/git.js', () => ({ getCoreExcludesFilePath: vi.fn(), @@ -24,6 +33,25 @@ afterEach(async () => { }); describe('watch path selection', () => { + it('admits the requested family budget and pending embeddings for watch refreshes', async () => { + vi.mocked(analysisMayEmbed).mockResolvedValueOnce(true); + const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; + const result = await resolveWatchOptions( + repoPath, + { resourceBudget: '4096', workers: '2', branch: 'feature' }, + baseline, + ); + expect(analysisMayEmbed).toHaveBeenLastCalledWith(repoPath, { branch: 'feature' }); + expect(resolveHostAnalysisResources).toHaveBeenLastCalledWith( + expect.objectContaining({ + totalMb: 4096, + workers: 2, + includeEmbeddings: true, + mainHeapMb: expect.any(Number), + }), + ); + expect(result).toMatchObject({ graphPublication: 'auto', resources: { mainHeapMb: 512 } }); + }); it('forwards and removes route configuration on subsequent refreshes', async () => { const aliases = [{ importPath: 'example.com/core', type: 'Engine', framework: 'gin' }]; const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; From 1aab86a6ceed169d56890e4012b69f6422cb910c Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 08:25:28 +0000 Subject: [PATCH 29/67] fix(analyze): close recovery and ownership gaps Retain interrupted private work, clear stale query widths, and keep native reservations until confirmed worker exit. Keep shared-member admission gates outside garbage-collected directories for flat and branch writers. Reject invalid routes and overlaps, and distinguish validation from uncertain writes. Make test imports work before build, isolate host-capacity assertions, and give legacy vector fixtures the production native pool allocation. Focused regressions, clean-build types, formatting and lint pass. Graph analysis maps 51 symbols across 21 paths. Two import-only test paths remain unmapped; direct review and integration tests cover them. The parent full suite passed 32,084 tests with five failures: two fixed legacy-vector fixture cases and three recovery cases awaiting isolation. Final full-suite and performance acceptance remain pending on the draft. Refs #3598 --- .../src/core/embeddings/character-chunk.ts | 7 +- .../src/core/ingestion/workers/worker-pool.ts | 21 +- gitnexus/src/core/run-analyze.ts | 2 +- gitnexus/src/mcp/local/local-backend.ts | 10 +- gitnexus/src/storage/analysis-slot-lock.ts | 46 +++- gitnexus/src/storage/embedding-job-journal.ts | 11 +- gitnexus/src/storage/index-capabilities.ts | 4 +- .../src/storage/shared-store-lifecycle.ts | 199 ++++++++++-------- gitnexus/src/storage/shared-store.ts | 10 +- .../analysis-embedding-flow.test.ts | 4 +- .../analysis-publication-flow.test.ts | 6 +- .../legacy-vector-reuse-index.test.ts | 4 +- .../integration/shared-store-adoption.test.ts | 24 ++- .../unit/analyze-worker-pool-size.test.ts | 7 + .../test/unit/embedding-job-journal.test.ts | 28 +++ .../unit/embedding-streaming-text.test.ts | 7 + .../local-backend-vector-generation.test.ts | 21 ++ .../unit/shared-store-generations.test.ts | 80 ++++++- .../vector-generation-preservation.test.ts | 18 ++ .../unit/worker-pool-timeout-retire.test.ts | 33 +++ gitnexus/vitest.config.ts | 3 + 21 files changed, 417 insertions(+), 128 deletions(-) diff --git a/gitnexus/src/core/embeddings/character-chunk.ts b/gitnexus/src/core/embeddings/character-chunk.ts index 0d0743264..e6c3b7b0d 100644 --- a/gitnexus/src/core/embeddings/character-chunk.ts +++ b/gitnexus/src/core/embeddings/character-chunk.ts @@ -19,7 +19,12 @@ export function* iterateCharacterChunks( chunkSize = 1200, overlap = 120, ): Generator { - if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || !Number.isSafeInteger(overlap)) + if ( + !Number.isSafeInteger(chunkSize) || + chunkSize < 1 || + !Number.isSafeInteger(overlap) || + overlap < 0 + ) throw new Error('Invalid character chunk size or overlap'); if (content.length <= chunkSize) { yield { diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index d748815a3..aebd4a0da 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -1595,15 +1595,24 @@ export const createWorkerPool = ( } // Recovery must settle before dispatch returns, but a failed thread // may never acknowledge termination. Bound that wait as in shutdown. - const termination = existing.terminate().then( - () => { - pendingWorkerReleases.delete(termination); - }, - () => undefined, - ); + let completeRelease!: () => void; + const termination = new Promise((resolve) => { + completeRelease = resolve; + }); + const released = () => { + existing.off('exit', released); + pendingWorkerReleases.delete(termination); + completeRelease(); + }; if (requireDrain) { pendingWorkerReleases.add(termination); + // A rejected terminate request is not proof of release. An eventual + // physical exit is, even when the termination promise rejected. + existing.once('exit', released); } + void existing.terminate().then(released, () => { + if (!requireDrain) released(); + }); if (!(await settledWithin(termination, poolOptions.shutdownDrainMs))) { existing.unref?.(); logger.warn( diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 227ecfbbf..a958f995e 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1323,7 +1323,7 @@ async function hasRetainedAnalysisWork(root: string): Promise { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } const meta = await loadMeta(slot); - return Boolean(meta?.embeddingJob || meta?.embeddingCheckpoint); + return Boolean(meta?.incrementalInProgress || meta?.embeddingJob || meta?.embeddingCheckpoint); }; if (await pending(root)) return true; let branches; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 2cdeca6ed..0c1423217 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -4591,9 +4591,15 @@ export class LocalBackend { // call's width, and the catch below must only clear an entry it did not set. let embeddedDims: number | undefined; try { - if (repo.generationUnknown) return []; + if (repo.generationUnknown) { + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + return []; + } const metadata = await this.readToolMetadata(repo); - if (repo.generationUnknown) return []; + if (repo.generationUnknown) { + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + return []; + } const selection = selectVectorGeneration( path.dirname(repo.lbugPath), metadata, diff --git a/gitnexus/src/storage/analysis-slot-lock.ts b/gitnexus/src/storage/analysis-slot-lock.ts index a02beb5a7..37c4ccccb 100644 --- a/gitnexus/src/storage/analysis-slot-lock.ts +++ b/gitnexus/src/storage/analysis-slot-lock.ts @@ -6,23 +6,47 @@ import { type IndexLockHandle, } from './index-lock.js'; import { storeRootOfCheckoutSlot } from './shared-store.js'; +import { getGlobalDir } from './global-dir.js'; -/** Admit branch writers under the member gate used by shared-store GC. Once - * admitted, the branch lock protects their files without serializing siblings. - * GC holds the gate while listing and trying branch locks, so a new branch - * cannot start writing between that listing and deletion of its member. */ +/** File-backed admission locks must outlive deletion of both member and store. + * Keep their empty parent directories: deleting a held gate lets a second + * process acquire a different lock at the same pathname. */ +export function sharedMemberGatePath(storeRoot: string, member: string): string { + // Store keys are unique within GITNEXUS_HOME. Keep the names as path + // components so filesystem aliases and Windows casing share the same gate. + return path.join( + getGlobalDir(), + 'locks', + 'shared-members', + path.basename(storeRoot), + path.basename(member), + ); +} + +/** Admit flat and branch writers under the member gate used by shared-store GC. + * Once admitted, their index locks protect files without serializing siblings. + * GC holds the persistent gate through member, graph, and store deletion. */ export async function acquireAnalysisSlotLock( directory: string, options: AcquireOptions = {}, ): Promise { - const branches = path.dirname(path.resolve(directory)); - const member = path.dirname(branches); - if (path.basename(branches) !== 'branches' || !storeRootOfCheckoutSlot(member)) - return acquireIndexLock(directory, options); - const gate = await acquireIndexLock(member, { ...options, sweep: false }); + const slot = path.resolve(directory); + const branches = path.dirname(slot); + const member = path.basename(branches) === 'branches' ? path.dirname(branches) : slot; + const storeRoot = storeRootOfCheckoutSlot(member); + if (!storeRoot) return acquireIndexLock(directory, options); + const gatePath = sharedMemberGatePath(storeRoot, member); + const gate = await acquireIndexLock(gatePath, { ...options, sweep: false }); try { - requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${member}.`); - return await acquireIndexLock(directory, options); + requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${gatePath}.`); + if (slot === member) return await acquireIndexLock(directory, options); + const memberLock = await acquireIndexLock(member, { ...options, sweep: false }); + try { + requireExclusiveIndexLock(memberLock, `Cannot acquire the shared member lock at ${member}.`); + return await acquireIndexLock(directory, options); + } finally { + memberLock.release(); + } } finally { gate.release(); } diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts index 0025606b7..58fa75e8c 100644 --- a/gitnexus/src/storage/embedding-job-journal.ts +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -281,13 +281,12 @@ export class EmbeddingJobJournal { const next = clone(this.state); await action(next); validateState(next, next.identity, this.manifest); + // A rejected rename/fsync may already have changed durable state. Pure + // validation failures before the first write leave the journal usable. + this.uncertain = true; await writeEmbeddingJobRecord(this.directory, 'job.json', next); this.state = next; - } catch (error) { - // Even a failed fsync can follow a successful rename. Never keep writing - // using an in-memory cursor that may differ from the durable one. - this.uncertain = true; - throw error; + this.uncertain = false; } finally { this.busy = false; } @@ -328,6 +327,7 @@ export class EmbeddingJobJournal { ) throw new Error('Invalid embedding unit acknowledgement'); // If the process dies after unlink, the still-active intent owns replay. + this.uncertain = true; await fs.rm(path.join(this.directory, filename(active.start.unit)), { force: true }); await syncEmbeddingJobDirectory(this.directory); if (result.reusedNodes === active.nodes) next.reusedUnits++; @@ -354,6 +354,7 @@ export class EmbeddingJobJournal { cause, attempt: this.attempt, }; + this.uncertain = true; await writeEmbeddingJobRecord(this.directory, filename(active.start.unit), record); next.failedUnits++; next.failedNodes += active.nodes; diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts index 9475d3a28..8f7656145 100644 --- a/gitnexus/src/storage/index-capabilities.ts +++ b/gitnexus/src/storage/index-capabilities.ts @@ -1,7 +1,7 @@ /** Published capability receipts, gated by current routing and artifact ownership. * Runtime query probes may further degrade these facts; status never loads a DB. */ -import path from 'node:path'; import type { RepoMeta } from './repo-meta.js'; +import { resolveGraphPath } from './shared-store.js'; import { assertOwnedVectorGeneration, selectVectorGeneration } from './vector-generation.js'; import { getExactScanLimit } from '../core/platform/capabilities.js'; @@ -17,7 +17,7 @@ export async function readIndexCapabilities( const selection = selectVectorGeneration( directory, metadata, - path.resolve(directory, metadata?.graphPath ?? 'lbug'), + resolveGraphPath(directory, metadata), exactScanLimit, ); if (selection.kind === 'ready') { diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index 4e240d8fa..bc4d5b51d 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -13,6 +13,7 @@ import { existsSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { acquireIndexLock, requireExclusiveIndexLock, type IndexLockHandle } from './index-lock.js'; +import { sharedMemberGatePath } from './analysis-slot-lock.js'; import { canonicalizePath, findRegistryEntryByRepoPath, @@ -197,108 +198,128 @@ export const reclaimSharedStoreLocked = async ( const checkoutsDir = path.join(storeRoot, 'checkouts'); const commitsDir = path.join(storeRoot, 'commits'); const referenced = new Set(); - let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name)); - if (opts.gc) { - const orphans = await orphanMembers(slots); - for (const slot of [...orphans]) { - // An analyze holds its slot's index lock until it has registered the - // checkout, so a slot that is seeded but not yet registered is busy, - // not orphaned. Judge (and, unless previewing, delete) only a slot - // whose lock is free, so the preview matches what --force would do. - let lock: IndexLockHandle; - try { - // Recheck durable work under ownership before deleting anything. In - // particular, acquiring the lock must not sweep a just-prepared graph. - lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: false }); - } catch { - orphans.delete(slot); - continue; - } - const branchLocks: IndexLockHandle[] = []; - try { - let busy = lock.lockFree; - if (!busy) { - for (const branch of (await memberMetadataDirectories(slot)).slice(1)) { - try { - const branchLock = await acquireIndexLock(branch, { timeoutMs: 1, sweep: false }); - branchLocks.push(branchLock); - if (branchLock.lockFree) { - busy = true; - break; - } - } catch { - busy = true; - break; - } - } - } - if (busy || !(await orphanMembers([slot])).has(slot)) { + const heldMemberGates: IndexLockHandle[] = []; + try { + let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name)); + if (opts.gc) { + const orphans = await orphanMembers(slots); + for (const slot of [...orphans]) { + // An analyze holds its slot's index lock until it has registered the + // checkout, so a slot that is seeded but not yet registered is busy, + // not orphaned. Judge (and, unless previewing, delete) only a slot + // whose lock is free, so the preview matches what --force would do. + let lock: IndexLockHandle; + let gate: IndexLockHandle | undefined; + try { + gate = await acquireIndexLock(sharedMemberGatePath(storeRoot, slot), { + timeoutMs: 1, + sweep: false, + }); + requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${slot}.`); + // Recheck durable work under ownership before deleting anything. In + // particular, acquiring the lock must not sweep a just-prepared graph. + lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: false }); + } catch { + gate?.release(); orphans.delete(slot); continue; } - if (!opts.dryRun) { - try { - await fs.rm(slot, { recursive: true, force: true }); - } catch { - // Like an undeletable graph below: keep it for the next collection - // rather than abort every store after this one. It stays a member, - // so whatever graph it still names is kept too. + const branchLocks: IndexLockHandle[] = []; + let dropped = false; + let deleting = false; + try { + let busy = lock.lockFree; + if (!busy) { + for (const branch of (await memberMetadataDirectories(slot)).slice(1)) { + try { + const branchLock = await acquireIndexLock(branch, { timeoutMs: 1, sweep: false }); + branchLocks.push(branchLock); + if (branchLock.lockFree) { + busy = true; + break; + } + } catch { + busy = true; + break; + } + } + } + if (busy || !(await orphanMembers([slot])).has(slot)) { orphans.delete(slot); - result.keptMembers.push(slot); continue; } + if (!opts.dryRun) { + try { + deleting = true; + await fs.rm(slot, { recursive: true, force: true }); + } catch { + // Like an undeletable graph below: keep it for the next collection + // rather than abort every store after this one. It stays a member, + // so whatever graph it still names is kept too. + orphans.delete(slot); + result.keptMembers.push(slot); + continue; + } + } + result.droppedMembers.push(slot); + dropped = true; + } finally { + for (const branchLock of branchLocks) branchLock.release(); + lock.release(); + // A deleted member must not be recreated until all graph and store + // deletion finishes. This gate is outside the recursively removed tree. + if (dropped || deleting) heldMemberGates.push(gate); + else gate.release(); } - result.droppedMembers.push(slot); - } finally { - for (const branchLock of branchLocks) branchLock.release(); - lock.release(); + } + slots = slots.filter((slot) => !orphans.has(slot)); + } + for (const slot of slots) { + for (const directory of await memberMetadataDirectories(slot)) { + const graphPath = (await loadMetaStrict(directory))?.graphPath; + if (graphPath) referenced.add(path.dirname(path.resolve(directory, graphPath))); } } - slots = slots.filter((slot) => !orphans.has(slot)); - } - for (const slot of slots) { - for (const directory of await memberMetadataDirectories(slot)) { - const graphPath = (await loadMetaStrict(directory))?.graphPath; - if (graphPath) referenced.add(path.dirname(path.resolve(directory, graphPath))); - } - } - for (const name of await listDirStrict(commitsDir)) { - const dir = path.join(commitsDir, name); - if (referenced.has(dir)) continue; - if (opts.dryRun) { - if (!name.startsWith('.')) result.removed.push(dir); - continue; + for (const name of await listDirStrict(commitsDir)) { + const dir = path.join(commitsDir, name); + if (referenced.has(dir)) continue; + if (opts.dryRun) { + if (!name.startsWith('.')) result.removed.push(dir); + continue; + } + try { + await fs.rm(dir, { recursive: true, force: true }); + if (!name.startsWith('.')) result.removed.push(dir); + } catch { + // Windows refuses to delete a file another process has open (an MCP + // reader). Keep it for the next reclaim instead of failing the caller. + if (!name.startsWith('.')) result.kept.push(dir); + } } - try { - await fs.rm(dir, { recursive: true, force: true }); - if (!name.startsWith('.')) result.removed.push(dir); - } catch { - // Windows refuses to delete a file another process has open (an MCP - // reader). Keep it for the next reclaim instead of failing the caller. - if (!name.startsWith('.')) result.kept.push(dir); - } - } - const isEmpty = async (): Promise => - (await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0; - if (!opts.dryRun && (await isEmpty())) { - // Also hold the cache lock (publish -> cache, the only nesting order) so - // a member saving caches cannot lose them, then re-check: a new member's - // slot may have appeared while waiting. The lock directories live inside - // the store; removing them while held is safe on POSIX and is retried on - // the next reclaim elsewhere. - await withStoreLock({ root: storeRoot }, 'cache', async () => { - if (!(await isEmpty())) return; - await fs - .rm(storeRoot, { recursive: true, force: true }) - .then(() => { - result.storeRemoved = true; - }) - .catch(() => {}); - }); + const isEmpty = async (): Promise => + (await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0; + if (!opts.dryRun && (await isEmpty())) { + // Also hold the cache lock (publish -> cache, the only nesting order) so + // a member saving caches cannot lose them, then re-check: a new member's + // slot may have appeared while waiting. The lock directories live inside + // the store; removing them while held is safe on POSIX and is retried on + // the next reclaim elsewhere. + await withStoreLock({ root: storeRoot }, 'cache', async () => { + if (!(await isEmpty())) return; + await fs + .rm(storeRoot, { recursive: true, force: true }) + .then(() => { + result.storeRemoved = true; + }) + .catch(() => {}); + }); + } + return result; + } finally { + for (const gate of heldMemberGates) gate.release(); } - return result; }; /** diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index f3c72c1b4..2ccfe67da 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -25,6 +25,7 @@ import { stripWindowsLongPathPrefix } from '../lib/utils.js'; import { getGlobalDir } from './global-dir.js'; import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from './storage-constants.js'; import { slotNameForCanonicalPath, STORAGE_PATH_ENV, STORAGE_ROOT_ENV } from './storage-slot.js'; +import type { RepoMeta } from './repo-meta.js'; export const SHARED_STORE_ENV = 'GITNEXUS_SHARED_STORE'; export const STORES_DIR = 'stores'; @@ -264,11 +265,16 @@ const readRecordedGraphPath = (slot: string): unknown => { } }; -export const resolveGraphPath = (storagePath: string): string => { +export const resolveGraphPath = ( + storagePath: string, + metadata?: Pick, +): string => { const own = path.join(storagePath, LBUG_DIRECTORY); let recorded: unknown; try { - recorded = readRecordedGraphPath(storagePath); + // Capability readers already loaded this receipt; validate that same + // snapshot without another stat, read, or parse on the status hot path. + recorded = metadata === undefined ? readRecordedGraphPath(storagePath) : metadata.graphPath; } catch { return own; } diff --git a/gitnexus/test/integration/analysis-embedding-flow.test.ts b/gitnexus/test/integration/analysis-embedding-flow.test.ts index 333cf8b3c..b4b2c16c7 100644 --- a/gitnexus/test/integration/analysis-embedding-flow.test.ts +++ b/gitnexus/test/integration/analysis-embedding-flow.test.ts @@ -4,8 +4,8 @@ import path from 'node:path'; import { createServer, type Server } from 'node:http'; import { execFileSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; -import { runFullAnalysis } from '../../dist/core/run-analyze.js'; -import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; import { loadMeta } from '../../src/storage/repo-manager.js'; import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; diff --git a/gitnexus/test/integration/analysis-publication-flow.test.ts b/gitnexus/test/integration/analysis-publication-flow.test.ts index 5d22c1343..dd0e1fe84 100644 --- a/gitnexus/test/integration/analysis-publication-flow.test.ts +++ b/gitnexus/test/integration/analysis-publication-flow.test.ts @@ -4,9 +4,9 @@ import path from 'node:path'; import { fork, execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; -import { continuePreparedAnalysis, runFullAnalysis } from '../../dist/core/run-analyze.js'; -import { verifyPublishedAnalysisGraph } from '../../dist/core/analysis-graph-verification.js'; -import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; +import { continuePreparedAnalysis, runFullAnalysis } from '../../src/core/run-analyze.js'; +import { verifyPublishedAnalysisGraph } from '../../src/core/analysis-graph-verification.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import { readPreparedAnalysis, diff --git a/gitnexus/test/integration/legacy-vector-reuse-index.test.ts b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts index 35b0f10b3..752b8fef7 100644 --- a/gitnexus/test/integration/legacy-vector-reuse-index.test.ts +++ b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts @@ -14,7 +14,9 @@ async function fixture() { directory, identity: 'b'.repeat(64), dimensions: 4, - bufferPoolBytes: 8 * 1024 * 1024, + // Match the production vector pool; LadybugDB 0.22 needs more than 8 MiB + // for the receipt tables and bounded insert batches. + bufferPoolBytes: 64 * 1024 * 1024, threads: 1, batchBytes: 1024 * 1024, }; diff --git a/gitnexus/test/integration/shared-store-adoption.test.ts b/gitnexus/test/integration/shared-store-adoption.test.ts index 9c7485f5e..bfeaa8d62 100644 --- a/gitnexus/test/integration/shared-store-adoption.test.ts +++ b/gitnexus/test/integration/shared-store-adoption.test.ts @@ -3,7 +3,7 @@ import { existsSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import { readSharedStorePointer, resolveSharedStore, @@ -116,6 +116,28 @@ describe('shared store adoption and reporting (#3352)', () => { ); }, 240_000); + it.each(['flat', 'branch'])( + 'retains unfinished private %s analysis before shared-store adoption', + async (kind) => { + await legacyIndex(wt); + const local = path.join(wt, '.gitnexus'); + const meta = await loadMeta(local); + if (!meta) throw new Error('Missing local fixture metadata'); + const directory = kind === 'flat' ? local : path.join(local, 'branches', 'unfinished'); + await fs.mkdir(directory, { recursive: true }); + await saveMeta(directory, { + ...meta, + incrementalInProgress: { startedAt: Date.now(), toWriteCount: 1 }, + }); + await analyze(wt); + expect((await listRegisteredRepos()).find((entry) => entry.path === wt)?.storagePath).toBe( + local, + ); + expect(readSharedStorePointer(wt)).toBeNull(); + }, + 240_000, + ); + it('status reports the shared graph and the leftover index with its removal command', async () => { await legacyIndex(wt); await analyze(wt); diff --git a/gitnexus/test/unit/analyze-worker-pool-size.test.ts b/gitnexus/test/unit/analyze-worker-pool-size.test.ts index b0f07655d..a9e2c8b61 100644 --- a/gitnexus/test/unit/analyze-worker-pool-size.test.ts +++ b/gitnexus/test/unit/analyze-worker-pool-size.test.ts @@ -2,6 +2,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const runFullAnalysisMock = vi.fn(); +// This suite checks option forwarding, not the host's capacity to run 12 +// workers. Resource admission has its own constrained-host tests. +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), + effectiveRamBytes: () => 32 * 1024 ** 3, +})); + vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: runFullAnalysisMock, })); diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index 30834aa25..11d0e66d3 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -47,6 +47,34 @@ afterEach(async () => { }); describe('durable embedding scheduling', () => { + it('keeps the journal usable after validation fails before any write', async () => { + const journal = await job(); + await expect(journal.indexing()).rejects.toThrow('accounting'); + await journal.begin(unit(0)); + await expect(journal.acknowledge({ reusedNodes: 3, chunks: 2 })).rejects.toThrow( + 'acknowledgement', + ); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + await expect(journal.fail('provider')).rejects.toThrow('Invalid embedding failure'); + await journal.begin(unit(1)); + expect(journal.snapshot()).toMatchObject({ reusedUnits: 1, active: unit(1) }); + }); + + it('requires reopening after a failed-unit write succeeds but its receipt fails', async () => { + const journal = await job(); + await journal.begin(unit(0)); + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + await rename(from, to); + if (path.basename(String(to)) === 'failed-0000000000000000.json') + throw new Error('injected failure after failed-unit rename'); + }); + await expect(journal.fail('provider')).rejects.toThrow('injected'); + await expect(journal.acknowledge({ reusedNodes: 0, chunks: 2 })).rejects.toThrow( + 'durability uncertain', + ); + }); + it('refuses foreign failed-unit identity instead of resetting another job', async () => { const journal = await job(); await journal.begin(unit(0)); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts index f193a83cc..2b513ce1b 100644 --- a/gitnexus/test/unit/embedding-streaming-text.test.ts +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -16,6 +16,13 @@ const fingerprintNode = (label: string, content: string) => ({ }); describe('bounded embedding text preserves existing vectors', () => { + it.each(['short', 'x'.repeat(2401)])( + 'rejects a negative overlap without skipping source text', + (content) => { + expect(() => [...iterateCharacterChunks(content, 1, 1, 1200, -1)]).toThrow('overlap'); + }, + ); + it.each([ [ 'Function', diff --git a/gitnexus/test/unit/local-backend-vector-generation.test.ts b/gitnexus/test/unit/local-backend-vector-generation.test.ts index 4db6b30d2..4025cf8de 100644 --- a/gitnexus/test/unit/local-backend-vector-generation.test.ts +++ b/gitnexus/test/unit/local-backend-vector-generation.test.ts @@ -34,6 +34,8 @@ interface Reader { limit: number, ): Promise; lastObservedPoolState: Map; + lastQueryEmbeddingDims: Map; + readToolMetadata(repo: { generationUnknown?: boolean }): Promise; } beforeEach(() => { vi.clearAllMocks(); @@ -42,6 +44,25 @@ beforeEach(() => { }); describe('MCP generation gate', () => { + it.each(['before-read', 'during-read'])( + 'clears a prior query width when ownership is lost %s', + async (when) => { + const backend = new LocalBackend() as unknown as Reader; + backend.lastQueryEmbeddingDims.set(graphPath, 4); + const repo = { lbugPath: graphPath, generationUnknown: when === 'before-read' }; + if (when === 'during-read') { + vi.spyOn(backend, 'readToolMetadata').mockImplementation(async (handle) => { + handle.generationUnknown = true; + return metadata; + }); + } + expect(await backend.semanticSearch(repo, 'hello', 5)).toEqual([]); + expect(backend.lastQueryEmbeddingDims.has(graphPath)).toBe(false); + expect(mocks.embed).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }, + ); + it.each(['pending', 'foreign-metadata', 'unknown-handle'])( 'skips model and table access for %s', async (state) => { diff --git a/gitnexus/test/unit/shared-store-generations.test.ts b/gitnexus/test/unit/shared-store-generations.test.ts index 5ab5e4acc..16316919c 100644 --- a/gitnexus/test/unit/shared-store-generations.test.ts +++ b/gitnexus/test/unit/shared-store-generations.test.ts @@ -1,4 +1,5 @@ -import { execFileSync } from 'node:child_process'; +import { execFile, execFileSync } from 'node:child_process'; +import { promisify } from 'node:util'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -21,7 +22,10 @@ import { } from '../../src/storage/shared-store.js'; import type { RepoMeta } from '../../src/storage/repo-meta.js'; import { acquireIndexLock } from '../../src/storage/index-lock.js'; -import { acquireAnalysisSlotLock } from '../../src/storage/analysis-slot-lock.js'; +import { + acquireAnalysisSlotLock, + sharedMemberGatePath, +} from '../../src/storage/analysis-slot-lock.js'; it('reports named slot-local generations as private and commit graphs as shared', () => { const slot = path.resolve('store/checkouts/owner'); @@ -195,6 +199,78 @@ describe('shared-store logical generations', () => { } }); + it('uses the same file-backed admission gate through a storage-home alias', async () => { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + const alias = path.join(root, 'home-alias'); + await fs.symlink(path.join(root, 'home'), alias, 'junction'); + const gate = await acquireIndexLock(sharedMemberGatePath(layout.root, slot), { sweep: false }); + try { + vi.stubEnv('GITNEXUS_HOME', alias); + const branch = path.join( + alias, + 'stores', + path.basename(layout.root), + 'checkouts', + path.basename(slot), + 'branches', + 'aliased', + ); + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1, sweep: false }).catch( + () => null, + ); + writer?.release(); + expect(writer).toBeNull(); + } finally { + gate.release(); + } + }); + + it.each([ + { stage: 'member', kind: 'flat' }, + { stage: 'graph', kind: 'flat' }, + { stage: 'store', kind: 'flat' }, + { stage: 'member', kind: 'branch' }, + { stage: 'graph', kind: 'branch' }, + { stage: 'store', kind: 'branch' }, + ])('keeps $kind admission closed throughout orphan $stage deletion', async ({ stage, kind }) => { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + const commit = path.join(layout.commitsDir, 'unreferenced'); + await fs.mkdir(commit, { recursive: true }); + await fs.writeFile(path.join(commit, 'lbug'), 'old graph'); + const branch = kind === 'flat' ? slot : path.join(slot, 'branches', 'new-branch'); + const remove = fs.rm.bind(fs); + let checked = false; + let admission: string | undefined; + vi.spyOn(fs, 'rm').mockImplementation(async (target, options) => { + await remove(target, options); + const deleting = stage === 'member' ? slot : stage === 'graph' ? commit : layout.root; + if (String(target) !== deleting) return; + checked = true; + // A second process must consult the lock on disk, not this process's + // in-memory lock queue, after GC has removed the old member directory. + const moduleUrl = new URL('../../src/storage/analysis-slot-lock.ts', import.meta.url).href; + const { stdout } = await promisify(execFile)(process.execPath, [ + '--import', + 'tsx', + '--input-type=module', + '--eval', + `import { acquireAnalysisSlotLock } from ${JSON.stringify(moduleUrl)}; + const lock = await acquireAnalysisSlotLock(${JSON.stringify(branch)}, { timeoutMs: 1, sweep: false }).catch(() => null); + console.log(lock ? 'admitted' : 'blocked'); + lock?.release();`, + ]); + admission = stdout.trim(); + }); + const result = await reclaimSharedStore(layout.root, { gc: true }); + expect(checked).toBe(true); + expect(admission).toBe('blocked'); + expect(result.droppedMembers).toEqual([slot]); + expect(result.storeRemoved).toBe(true); + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1000, sweep: false }); + writer.release(); + }); + it.each(['e'.repeat(64), null])( 'retains an orphan with historical vector evidence: %s', async (source) => { diff --git a/gitnexus/test/unit/vector-generation-preservation.test.ts b/gitnexus/test/unit/vector-generation-preservation.test.ts index 650d3e7f8..7b904a47f 100644 --- a/gitnexus/test/unit/vector-generation-preservation.test.ts +++ b/gitnexus/test/unit/vector-generation-preservation.test.ts @@ -63,6 +63,24 @@ afterEach(async () => { }); describe('preserved vector coverage', () => { + it.each(['../foreign/lbug', '/foreign/lbug', 'lbug.generation.invalid'])( + 'does not report vectors as ready for a rejected graph route: %s', + async (graphPath) => { + const f = await fixture(); + const capabilities = await readIndexCapabilities(f.root, { ...f.metadata, graphPath }); + expect(capabilities?.vectorSearch).toMatchObject({ status: 'unavailable' }); + }, + ); + + it('does not report vectors as ready through a symlinked graph generation', async () => { + const f = await fixture(); + await fs.rename(f.graph, f.graph + '.foreign'); + await fs.symlink(f.graph + '.foreign', f.graph); + expect((await readIndexCapabilities(f.root, f.metadata))?.vectorSearch).toMatchObject({ + status: 'unavailable', + }); + }); + it('advances two staged graphs with a small historical reference and no vector writes', async () => { const f = await fixture(); const before = await fs.stat(f.vectors); diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index bfa170b18..3dad9e0e9 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -94,6 +94,39 @@ afterEach(() => { }); describe('worker pool timeout retirement', () => { + it('retains a rejected termination reservation until physical exit, then releases it', async () => { + let broken: TimeoutThenHealthyWorker | undefined; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + shutdownDrainMs: 20, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id === 0) { + broken = worker; + worker.postMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'sub-batch') + queueMicrotask(() => worker.emit('error', new Error('worker failed'))); + }; + worker.terminate = async () => { + worker.terminateCalls++; + throw new Error('termination rejected'); + }; + } + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.dispatch([{ path: 'src/error.ts', content: '' }])).rejects.toThrow(); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + broken!.emit('exit', 1); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + broken?.emit('exit', 1); + await pool.terminate().catch(() => {}); + } + }); + it('refuses replacement and phase handoff while a budgeted native worker has not drained', async () => { const pool = createWorkerPool(workerUrl, 1, { requireDrain: true, diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index f2ac1bbd6..1db21a4fd 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -3,6 +3,7 @@ import PerfSequencer from './test/helpers/perf-sequencer.js'; // Keep native tests in exactly one project (TESTING.md). const EMBEDDING_VECTOR_STORE_TEST = 'test/integration/embedding-vector-store.test.ts'; +const LEGACY_VECTOR_REUSE_TEST = 'test/integration/legacy-vector-reuse-index.test.ts'; export default defineConfig({ test: { @@ -82,6 +83,7 @@ export default defineConfig({ name: 'lbug-db', include: [ EMBEDDING_VECTOR_STORE_TEST, + LEGACY_VECTOR_REUSE_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', @@ -201,6 +203,7 @@ export default defineConfig({ include: ['test/**/*.test.ts'], exclude: [ EMBEDDING_VECTOR_STORE_TEST, + LEGACY_VECTOR_REUSE_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', From 3c07998919049fe3cde2d2bead8834c286d32d59 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 09:14:16 +0000 Subject: [PATCH 30/67] perf(embeddings): batch reused vector writes Avoid a native transaction per reused node while retaining the byte budget and durable unit boundary. Cover mixed inputs, cancellation, and replay after read or committed-write failures. Native regression fails before the fix (65 commits for 64 nodes). 79 focused tests and all three former recovery timeouts pass. Fresh graph gate: 18 symbols, 2 files, 36 affected flows; complete mapping. Final whole-command performance acceptance remains pending. Refs #3598 --- .../core/embeddings/embedding-job-runner.ts | 74 ++++++++++++++----- .../embedding-vector-store.test.ts | 66 +++++++++++++++++ 2 files changed, 120 insertions(+), 20 deletions(-) diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index 1f938ee38..19b8ae56b 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -142,12 +142,28 @@ export async function runEmbeddingJob( const releaseScratch = budget.reserve(scratchBytes); let batch: Array & { text: string; release: () => void }> = []; + let copyBatch: VectorWriteChunk[] = []; + let copyBytes = 0; + let copyReleases: Array<() => void> = []; const generated = new Map(); const failed = new Set(); let failureCause: 'input' | 'provider' = 'input'; let abort: unknown; let reusedNodes = complete.size; + const flushCopies = async (): Promise => { + if (!copyBatch.length) return; + signal?.throwIfAborted(); + try { + await store.appendBatch(copyBatch); + } finally { + for (const release of copyReleases) release(); + copyBatch = []; + copyBytes = 0; + copyReleases = []; + } + }; + const flush = async (): Promise => { if (!batch.length) return; signal?.throwIfAborted(); @@ -265,31 +281,42 @@ export async function runEmbeddingJob( // Flush before copy so there is only one vector batch in memory. await flush(); if (abort) break; - const copyRows = Math.min( - 256, - Math.floor( - Math.min( - reuse.options.batchBytes, - store.options.batchBytes, - budget.limit - budget.usedBytes, - ) / - (32 * dimensions + 1024 + 4 * node.id.length), - ), - ); - if (copyRows < 1) - throw new Error('Reusable vector batch does not fit the resource budget'); - for (let start = 0; start < reusable.chunks; start += copyRows) { + const copyRowBytes = 32 * dimensions + 1024 + 4 * node.id.length; + for (let start = 0; start < reusable.chunks; ) { + // Share one bounded transaction across neighboring reused nodes. + // Retain its reservation until the write finishes, including while + // validating the next node's historical input. + if ( + copyBatch.length && + (copyBatch.length === 256 || + Math.min(store.options.batchBytes - copyBytes, budget.limit - budget.usedBytes) < + copyRowBytes) + ) + await flushCopies(); + const copyRows = Math.min( + 256 - copyBatch.length, + Math.floor( + Math.min( + reuse.options.batchBytes, + store.options.batchBytes - copyBytes, + budget.limit - budget.usedBytes, + ) / copyRowBytes, + ), + ); + if (copyRows < 1) + throw new Error('Reusable vector batch does not fit the resource budget'); const length = Math.min(copyRows, reusable.chunks - start); // Reuse has no HTTP body or JSON decoder; retain only vector copies. - const release = budget.reserve(length * (32 * dimensions + 1024 + 4 * node.id.length)); + const bytes = length * copyRowBytes; + const release = budget.reserve(bytes); try { const chunks = await reuse.readChunks(reusable, start, length); const delta = input.sourceStartLine - (reusable.sourceStartLine ?? 0); - await store.append( - node.id, - input.contentHash, - chunks.map((chunk) => ({ + copyBatch.push( + ...chunks.map((chunk) => ({ ...chunk, + nodeId: node.id, + contentHash: input.contentHash, startLine: chunk.startLine + delta, endLine: chunk.endLine === reusable.sourceEndLine @@ -297,14 +324,19 @@ export async function runEmbeddingJob( : chunk.endLine + delta, })), ); - } finally { + } catch (error) { release(); + throw error; } + copyReleases.push(release); + copyBytes += bytes; + start += length; } copied.push({ nodeId: node.id, contentHash: input.contentHash, chunks: reusable.chunks }); reusedNodes++; continue; } + if (copyBatch.length) await flushCopies(); if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { try { const names = await extractStructuralNames(node.content, node.filePath); @@ -369,6 +401,7 @@ export async function runEmbeddingJob( node.fieldNames = undefined; if (abort) break; } + if (copyBatch.length) await flushCopies(); if (!abort) await flush(); const finished = [...generated] .filter(([id, node]) => node.ended && !failed.has(id)) @@ -392,6 +425,7 @@ export async function runEmbeddingJob( if (abort) throw abort; } finally { for (const row of batch) row.release(); + for (const release of copyReleases) release(); releaseScratch(); fetched.release(); } diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index c8d41eaa0..0cd91719c 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -249,7 +249,9 @@ describe('bounded durable embedding runner', () => { it('completes copied groups together at the unit boundary without provider requests', async () => { const fixture = await runnerFixture(64); const finish = vi.spyOn(store, 'finishNodes'); + const queries = vi.spyOn(lbug.Connection.prototype, 'query'); const result = await fixture.run({ + queueBytes: 512 * 1024, reuse: { options: { batchBytes: 64 * 1024 }, async reusable(nodeId: string, contentHash: string) { @@ -263,8 +265,72 @@ describe('bounded durable embedding runner', () => { expect(fixture.provider).not.toHaveBeenCalled(); expect(result.state).toMatchObject({ reusedNodes: 64, completedNodes: 0, chunks: 64 }); expect(finish).toHaveBeenCalledTimes(1); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + // A complete unit must not pay one durable native commit per reused node. + expect(queries.mock.calls.filter(([sql]) => sql === 'COMMIT').length).toBeLessThan(10); }); + it('flushes copied vectors before generated inputs and preserves complete mixed coverage', async () => { + const fixture = await runnerFixture(65); + const result = await fixture.run({ + queueBytes: 512 * 1024, + reuse: { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return Number(nodeId.slice(-5)) < 33 + ? { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 } + : null; + }, + async readChunks() { + return [chunk(0)]; + }, + }, + }); + expect(result.state).toMatchObject({ reusedNodes: 33, completedNodes: 32, chunks: 65 }); + const submitted = fixture.provider.mock.calls.flatMap(([texts]) => texts); + expect(submitted).toHaveLength(32); + expect(new Set(submitted).size).toBe(32); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + }); + + it.each(['read', 'write', 'cancel'] as const)( + 'replays a copy %s interruption without duplicate chunks or provider requests', + async (fault) => { + const fixture = await runnerFixture(64); + let fail = true; + const controller = new AbortController(); + const append = store.appendBatch.bind(store); + vi.spyOn(store, 'appendBatch').mockImplementation(async (chunks) => { + await append(chunks); + if (fail && fault === 'write') throw new Error('injected reuse failure'); + }); + const reuse = { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 }; + }, + async readChunks(receipt: { nodeId: string }) { + if (fail && receipt.nodeId.endsWith('n00055')) { + if (fault === 'read') throw new Error('injected reuse failure'); + if (fault === 'cancel') controller.abort(new Error('injected reuse failure')); + } + return [chunk(0)]; + }, + }; + await expect( + fixture.run({ reuse, queueBytes: 512 * 1024, signal: controller.signal }), + ).rejects.toThrow('reuse failure'); + expect(fixture.journal.snapshot()).toMatchObject({ reusedNodes: 0, chunks: 0 }); + expect(fixture.journal.snapshot().active).not.toBeNull(); + await fixture.reopen(); + fail = false; + const result = await fixture.run({ reuse, queueBytes: 512 * 1024 }); + expect(result.state).toMatchObject({ reusedNodes: 64, completedNodes: 0, chunks: 64 }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + }, + ); + it('refuses a symlinked native shadow sidecar before opening the artifact', async () => { const options = await setup(); await store.close(); From d6f487f974d552522e87f5e8c4119030e5f1075f Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 09:46:33 +0000 Subject: [PATCH 31/67] fix(analyze): honor explicit worker overrides Preserve the existing CPU oversubscription override while reserving memory for every requested worker. Automatic concurrency stays CPU-bounded. Validate on one- and four-CPU hosts: 81 tests pass; the override regressions fail before the fix. --- gitnexus/src/core/analysis-resource-budget.ts | 8 +++----- .../unit/analysis-resource-budget.test.ts | 19 ++++++++++++++++--- .../unit/analyze-worker-pool-size.test.ts | 6 ++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 65971af62..021c774f6 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -150,11 +150,9 @@ export function resolveAnalysisResourceBudget( request.workers === undefined ? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers) : integer(request.workers, 'parser workers'); - if (includeParsing && desiredWorkers > cpuWorkers) { - throw new Error( - `Requested ${desiredWorkers} parser workers exceeds the ${cpuWorkers}-worker CPU allowance`, - ); - } + // Explicit worker counts intentionally permit CPU oversubscription, as the + // parser's existing override does. Reserve every worker's heap/native memory + // below; automatic concurrency remains bounded by the usable CPU count. const workerBudgetMb = totalMb - headroomMb - supervisorMb - graphPoolMb - mainHeapMb; const explicitWorkerHeapMb = request.workerHeapMb === undefined diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index e151df79b..466a9fa28 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -115,6 +115,21 @@ describe('analysis resource admission', () => { expect(Object.isFrozen(budget.embedding)).toBe(true); }); + it.each([1, 4])( + 'admits an explicit worker override on %s CPU(s) within its memory budget', + (availableCpus) => { + const input = { ...request, availableCpus, mainHeapMb: 8192, workers: 12 }; + const budget = resolveAnalysisResourceBudget(input); + expect(budget.parse.workers).toBe(12); + expect(budget.parse.workerHeapMb).toBeGreaterThanOrEqual(512); + expect(budget.parse.nativeThreads).toBe(availableCpus); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => resolveAnalysisResourceBudget({ ...input, workerHeapMb: 4096 })).toThrow( + 'do not fit', + ); + }, + ); + it('admits an HTTP embedding-only resume below the parser minimum', () => { const input = { ...request, totalMb: 1024, modelMemoryMb: 0 }; expect(() => resolveAnalysisResourceBudget(input)).toThrow('cannot admit'); @@ -167,9 +182,7 @@ describe('analysis resource admission', () => { expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 4, workerHeapMb: 4096 }), ).toThrow('do not fit'); - expect(() => resolveAnalysisResourceBudget({ ...request, workers: 24 })).toThrow( - 'CPU allowance', - ); + expect(() => resolveAnalysisResourceBudget({ ...request, workers: 24 })).toThrow('do not fit'); expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: 0 })).toThrow( 'graph pool bytes', ); diff --git a/gitnexus/test/unit/analyze-worker-pool-size.test.ts b/gitnexus/test/unit/analyze-worker-pool-size.test.ts index a9e2c8b61..5d5b08f23 100644 --- a/gitnexus/test/unit/analyze-worker-pool-size.test.ts +++ b/gitnexus/test/unit/analyze-worker-pool-size.test.ts @@ -2,6 +2,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const runFullAnalysisMock = vi.fn(); +vi.mock('node:os', async (importOriginal) => { + const actual = await importOriginal(); + const mocked = { ...actual, availableParallelism: () => 4 }; + return { ...mocked, default: mocked }; +}); + // This suite checks option forwarding, not the host's capacity to run 12 // workers. Resource admission has its own constrained-host tests. vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ From 2c6dd97c645ea6eae3003bbef199d5496f7092ad Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 11:26:27 +0000 Subject: [PATCH 32/67] fix(storage): validate opened artifact files Validate descriptors before reading bounded receipts and manifests. Use nonblocking opens to reject FIFO substitutions and verify directory entries when O_NOFOLLOW is unavailable. Preserve the metadata cache fast path and guard generation paths explicitly. Remove the redundant pending-node fallback. Add race, symlink, cleanup, and cache regressions to platform coverage. --- gitnexus/scripts/cross-platform-tests.ts | 3 + gitnexus/src/core/analysis-parse-cache.ts | 24 ++-- .../embeddings/embedding-work-manifest.ts | 8 +- gitnexus/src/core/run-analyze.ts | 2 +- gitnexus/src/storage/embedding-job.ts | 44 +++++-- .../src/storage/legacy-embedding-source.ts | 10 +- gitnexus/src/storage/shared-store.ts | 27 ++++- .../unit/embedding-job-file-security.test.ts | 103 ++++++++++++++++ .../unit/shared-store-read-security.test.ts | 113 ++++++++++++++++++ 9 files changed, 294 insertions(+), 40 deletions(-) create mode 100644 gitnexus/test/unit/embedding-job-file-security.test.ts create mode 100644 gitnexus/test/unit/shared-store-read-security.test.ts diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index bcae19f36..f235d3400 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -26,6 +26,9 @@ // Platform-specific logic tests — contain explicit process.platform guards // or test behavior that differs across operating systems const PLATFORM_LOGIC = [ + // Descriptor validation differs when O_NOFOLLOW is unavailable on Windows. + 'test/unit/embedding-job-file-security.test.ts', + 'test/unit/shared-store-read-security.test.ts', 'test/unit/setup.test.ts', 'test/unit/setup-jsonc.test.ts', 'test/unit/setup-codex.test.ts', diff --git a/gitnexus/src/core/analysis-parse-cache.ts b/gitnexus/src/core/analysis-parse-cache.ts index a45c87178..9f10c1c37 100644 --- a/gitnexus/src/core/analysis-parse-cache.ts +++ b/gitnexus/src/core/analysis-parse-cache.ts @@ -1,6 +1,5 @@ /** Parser cache publication shared by an in-process writer and its exit supervisor. */ import fs from 'node:fs/promises'; -import { constants } from 'node:fs'; import path from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; import { @@ -11,7 +10,11 @@ import { type ParseCache, } from '../storage/parse-cache.js'; import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js'; -import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; +import { + embeddingJobDirectory, + openEmbeddingJobFile, + syncEmbeddingJobDirectory, +} from '../storage/embedding-job.js'; import { withStoreLock } from '../storage/shared-store-lifecycle.js'; import type { SharedStoreLayout } from '../storage/shared-store.js'; import type { RepoMeta } from '../storage/repo-meta.js'; @@ -193,26 +196,17 @@ export async function loadAnalysisParseCache( if (!generation) throw new Error('Cache publication requires a published graph generation'); root = await embeddingJobDirectory(root); const file = path.join(root, receiptName(generation)); - let stat; + let opened; try { - stat = await fs.lstat(file); + opened = await openEmbeddingJobFile(file); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; throw error; } - if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_HANDOFF_BYTES) - throw new Error('Unverified prepared parser cache receipt'); - const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const { handle, stat } = opened; let bytes: Buffer; try { - const opened = await handle.stat(); - if ( - !opened.isFile() || - opened.size !== stat.size || - opened.ino !== stat.ino || - opened.dev !== stat.dev - ) - throw new Error('Prepared parser cache receipt changed'); + if (stat.size > MAX_HANDOFF_BYTES) throw new Error('Unverified prepared parser cache receipt'); bytes = Buffer.alloc(stat.size + 1); let length = 0; while (length < bytes.length) { diff --git a/gitnexus/src/core/embeddings/embedding-work-manifest.ts b/gitnexus/src/core/embeddings/embedding-work-manifest.ts index 1c14018e7..9001d0e16 100644 --- a/gitnexus/src/core/embeddings/embedding-work-manifest.ts +++ b/gitnexus/src/core/embeddings/embedding-work-manifest.ts @@ -1,5 +1,4 @@ import { createHash } from 'node:crypto'; -import { constants } from 'node:fs'; import fs, { type FileHandle } from 'node:fs/promises'; import path from 'node:path'; import { assertValidGraphNodeId } from '../graph/node-identity.js'; @@ -7,6 +6,7 @@ import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; import { EMBEDDABLE_LABELS, type EmbeddableNode } from './types.js'; import { embeddingJobDirectory, + openEmbeddingJobFile, readEmbeddingJobRecord, syncEmbeddingJobDirectory, writeEmbeddingJobRecord, @@ -104,12 +104,8 @@ class BufferedLines { /** Framing stays bounded even for a corrupt file without a newline. */ async function* lines(file: string, start = 0): AsyncGenerator<{ text: string; end: number }> { - if (!(await fs.lstat(file)).isFile()) - throw new Error('Embedding manifest artifact must be a regular file'); - const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const { handle } = await openEmbeddingJobFile(file); try { - if (!(await handle.stat()).isFile()) - throw new Error('Embedding manifest artifact must be a regular file'); const buffer = Buffer.alloc(IO_BYTES); let pending = Buffer.alloc(0); let position = start; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index a958f995e..c99b681a5 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -6147,7 +6147,7 @@ async function runFullAnalysisInner( : {}), }, excluded: new Set([ - ...(pendingEmbeddingNodeIds ?? []), + ...pendingEmbeddingNodeIds, ...inheritedUnsafeNodeIds, ...restoreFailedNodeIds, ]), diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 42d292619..eae965ccb 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -1,7 +1,7 @@ /** Filesystem-only embedding receipts; importing this never loads a model or DB. */ import { createHash, randomUUID } from 'node:crypto'; -import { constants } from 'node:fs'; -import fs from 'node:fs/promises'; +import { constants, type Stats } from 'node:fs'; +import fs, { type FileHandle } from 'node:fs/promises'; import path from 'node:path'; const MAX_RECORD_BYTES = 32 * 1024; @@ -15,6 +15,40 @@ const RECORD_NAMES = new Set([ ]); const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; +/** Pin the file before checking it; a replaced pathname cannot redirect reads. */ +export async function openEmbeddingJobFile( + file: string, +): Promise<{ handle: FileHandle; stat: Stats }> { + let handle: FileHandle; + try { + handle = await fs.open( + file, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ELOOP') + throw new Error('Embedding artifact must be a regular file', { cause: error }); + throw error; + } + try { + const stat = await handle.stat(); + if (!stat.isFile()) throw new Error('Embedding artifact must be a regular file'); + // Windows lacks O_NOFOLLOW. Validate the directory entry against the + // pinned descriptor before reading any bytes; later path swaps are harmless. + if (!constants.O_NOFOLLOW) { + const entry = await fs.lstat(file); + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Embedding artifact must be a regular file'); + if (entry.dev !== stat.dev || entry.ino !== stat.ino) + throw new Error('Embedding artifact changed during open'); + } + return { handle, stat }; + } catch (error) { + await handle.close(); + throw error; + } +} + export async function embeddingJobDirectory(directory: string): Promise { const stat = await fs.lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) @@ -69,12 +103,8 @@ export async function readEmbeddingJobRecord(directory: string, name: string): P throw new Error('Invalid embedding receipt name'); const canonical = await embeddingJobDirectory(directory); const target = path.join(canonical, name); - if (!(await fs.lstat(target)).isFile()) - throw new Error('Embedding receipt must be a regular file'); - const handle = await fs.open(target, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const { handle } = await openEmbeddingJobFile(target); try { - if (!(await handle.stat()).isFile()) - throw new Error('Embedding receipt must be a regular file'); const buffer = Buffer.alloc(MAX_RECORD_BYTES + 1); let length = 0; while (length < buffer.length) { diff --git a/gitnexus/src/storage/legacy-embedding-source.ts b/gitnexus/src/storage/legacy-embedding-source.ts index c7be450ab..98cf9c803 100644 --- a/gitnexus/src/storage/legacy-embedding-source.ts +++ b/gitnexus/src/storage/legacy-embedding-source.ts @@ -2,13 +2,14 @@ * The caller holds the slot lock and retires/checkpoints the source before * recording it. No current configuration is used to fill historical fields. */ import { createHash, randomUUID } from 'node:crypto'; -import { constants, type BigIntStats } from 'node:fs'; +import type { BigIntStats } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; import type { EmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; import { assertValidGraphNodeId } from '../core/graph/node-identity.js'; import { embeddingJobDirectory, + openEmbeddingJobFile, readEmbeddingJobRecord, writeEmbeddingJobRecord, syncEmbeddingJobDirectory, @@ -222,12 +223,9 @@ async function* exclusions( expected: LegacyEmbeddingSourceRecord['excluded'], ): AsyncGenerator { const file = path.join(directory, 'excluded.jsonl'); - if (!(await fs.lstat(file)).isFile()) throw new Error('Legacy exclusions must be a regular file'); - const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const { handle, stat } = await openEmbeddingJobFile(file); try { - const stat = await handle.stat(); - if (!stat.isFile() || stat.size !== expected.bytes) - throw new Error('Legacy exclusions size mismatch'); + if (stat.size !== expected.bytes) throw new Error('Legacy exclusions size mismatch'); const buffer = Buffer.alloc(IO_BYTES), checksum = createHash('sha256'); let pending = Buffer.alloc(0), diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index 2ccfe67da..8c955ef96 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -245,16 +245,31 @@ const readRecordedGraphPath = (slot: string): unknown => { const metaPath = path.resolve(root, INDEX_METADATA_FILE); const rel = path.relative(root, metaPath); if (rel.startsWith('..') || path.isAbsolute(rel)) return undefined; - const current = fs.statSync(metaPath); const cached = recordedGraphCache.get(metaPath); - if (cached?.key === `${current.ino}:${current.size}:${current.mtimeMs}:${current.ctimeMs}`) - return cached.recorded; + if (cached) { + const current = fs.lstatSync(metaPath); + if (!current.isFile()) throw new Error('Index metadata must be a regular file'); + if ( + cached.key === + `${current.dev}:${current.ino}:${current.size}:${current.mtimeMs}:${current.ctimeMs}` + ) + return cached.recorded; + } // One open, then stat and read through the same descriptor, so the cache // key always describes the bytes that were parsed. - const fd = fs.openSync(metaPath, 'r'); + const fd = fs.openSync( + metaPath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0), + ); try { const stat = fs.fstatSync(fd); - const key = `${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; + if (!stat.isFile()) throw new Error('Index metadata must be a regular file'); + if (!fs.constants.O_NOFOLLOW) { + const entry = fs.lstatSync(metaPath); + if (!entry.isFile() || entry.ino !== stat.ino || entry.dev !== stat.dev) + throw new Error('Index metadata changed during open'); + } + const key = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; if (cached?.key === key) return cached.recorded; const recorded = (JSON.parse(fs.readFileSync(fd, 'utf-8')) as { graphPath?: unknown }) .graphPath; @@ -281,6 +296,8 @@ export const resolveGraphPath = ( if (typeof recorded !== 'string') return own; if (/^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { const graph = path.join(storagePath, recorded); + const relative = path.relative(path.resolve(storagePath), path.resolve(graph)); + if (relative.startsWith('..') || path.isAbsolute(relative)) return own; // Never follow an edited pointer to a symlink or another slot. A missing // committed generation must fail at open, rather than serve an older lbug. try { diff --git a/gitnexus/test/unit/embedding-job-file-security.test.ts b/gitnexus/test/unit/embedding-job-file-security.test.ts new file mode 100644 index 000000000..e302cb4a7 --- /dev/null +++ b/gitnexus/test/unit/embedding-job-file-security.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { constants } from 'node:fs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; + +const directories: string[] = []; +async function fixture(payload: unknown): Promise { + const directory = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), 'gn-receipt-security-')), + ); + directories.push(directory); + await writeEmbeddingJobRecord(directory, 'job.json', payload); + return directory; +} + +afterEach(async () => { + vi.restoreAllMocks(); + vi.doUnmock('node:fs'); + vi.resetModules(); + await Promise.all( + directories.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })), + ); +}); + +it.skipIf(process.platform === 'win32')( + 'rejects a receipt replaced by a FIFO without blocking its open', + async () => { + const directory = await fixture({ owned: true }); + const file = path.join(directory, 'job.json'); + const open = fs.open.bind(fs); + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + if (target === file) { + await fs.unlink(file); + execFileSync('mkfifo', [file]); + // Fail before invoking the blocking syscall on the regressed version. + if (typeof flags !== 'number' || !(flags & constants.O_NONBLOCK)) + throw new Error('Unsafe blocking open of FIFO'); + } + return open(target, flags, mode); + }); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('regular file'); + }, +); + +it('rejects a different regular file at the opened path when O_NOFOLLOW is unavailable', async () => { + const directory = await fixture({ owned: true }); + const replacement = await fixture({ replacement: true }); + const file = path.join(directory, 'job.json'); + vi.doMock('node:fs', async (original) => { + const actual = await original(); + return { ...actual, constants: { ...actual.constants, O_NOFOLLOW: undefined } }; + }); + vi.resetModules(); + const { readEmbeddingJobRecord: readWithoutNoFollow } = + await import('../../src/storage/embedding-job.js'); + const open = fs.open.bind(fs); + let close: ReturnType | undefined; + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + const handle = await open(target, flags, mode); + if (target === file) { + close = vi.spyOn(handle, 'close'); + await fs.rename(path.join(replacement, 'job.json'), file); + } + return handle; + }); + await expect(readWithoutNoFollow(directory, 'job.json')).rejects.toThrow('changed during open'); + expect(close).toHaveBeenCalledOnce(); +}); + +it.skipIf(process.platform === 'win32')( + 'rejects a symlink swapped in during open when O_NOFOLLOW is unavailable', + async () => { + const directory = await fixture({ owned: true }); + const outside = await fixture({ outside: true }); + const file = path.join(directory, 'job.json'); + vi.doMock('node:fs', async (original) => { + const actual = await original(); + return { ...actual, constants: { ...actual.constants, O_NOFOLLOW: undefined } }; + }); + vi.resetModules(); + const { readEmbeddingJobRecord: readWithoutNoFollow } = + await import('../../src/storage/embedding-job.js'); + const open = fs.open.bind(fs); + let close: ReturnType | undefined; + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + if (target === file) { + await fs.unlink(file); + await fs.symlink(path.join(outside, 'job.json'), file); + } + const handle = await open(target, flags, mode); + if (target === file) close = vi.spyOn(handle, 'close'); + return handle; + }); + await expect(readWithoutNoFollow(directory, 'job.json')).rejects.toThrow('regular file'); + expect(close).toHaveBeenCalledOnce(); + }, +); diff --git a/gitnexus/test/unit/shared-store-read-security.test.ts b/gitnexus/test/unit/shared-store-read-security.test.ts new file mode 100644 index 000000000..a8d814c4e --- /dev/null +++ b/gitnexus/test/unit/shared-store-read-security.test.ts @@ -0,0 +1,113 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { resolveGraphPath } from '../../src/storage/shared-store.js'; +import { INDEX_METADATA_FILE } from '../../src/storage/storage-constants.js'; + +const directories: string[] = []; +const first = `lbug.generation.${'a'.repeat(32)}`; +const second = `lbug.generation.${'b'.repeat(32)}`; + +function fixture() { + const slot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gn-graph-read-'))); + directories.push(slot); + const metadata = path.join(slot, INDEX_METADATA_FILE); + fs.writeFileSync(metadata, JSON.stringify({ graphPath: first })); + fs.writeFileSync(path.join(slot, first), 'first graph'); + fs.writeFileSync(path.join(slot, second), 'second graph'); + return { slot, metadata }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.doUnmock('fs'); + vi.resetModules(); + for (const directory of directories.splice(0)) + fs.rmSync(directory, { recursive: true, force: true }); +}); + +it('keeps warm metadata reads to one stat without opening or parsing the file', () => { + const { slot, metadata } = fixture(); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + const open = vi.spyOn(fs, 'openSync'); + const read = vi.spyOn(fs, 'readFileSync'); + const parse = vi.spyOn(JSON, 'parse'); + const stat = vi.spyOn(fs, 'lstatSync'); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + expect(open).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + expect(parse).not.toHaveBeenCalled(); + expect(stat.mock.calls.filter(([file]) => file === metadata)).toHaveLength(1); +}); + +it('reads the opened metadata snapshot and invalidates it after pathname replacement', () => { + const { slot, metadata } = fixture(); + const replacement = path.join(slot, 'replacement.json'); + fs.writeFileSync(replacement, JSON.stringify({ graphPath: second })); + const read = fs.readFileSync.bind(fs); + const spy = vi.spyOn(fs, 'readFileSync').mockImplementation((...args) => { + if (typeof args[0] === 'number') { + spy.mockRestore(); + fs.renameSync(replacement, metadata); + } + return read(...args); + }); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + expect(resolveGraphPath(slot)).toBe(path.join(slot, second)); +}); + +it.skipIf(process.platform === 'win32').each([false, true])( + 'rejects symlinked metadata with a warm cache of %s', + (warm) => { + const { slot, metadata } = fixture(); + const outside = fixture(); + if (warm) expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + fs.unlinkSync(metadata); + fs.symlinkSync(outside.metadata, metadata); + const read = vi.spyOn(fs, 'readFileSync'); + expect(resolveGraphPath(slot)).toBe(path.join(slot, 'lbug')); + expect(read).not.toHaveBeenCalled(); + }, +); + +it.skipIf(process.platform === 'win32')( + 'rejects a metadata symlink when O_NOFOLLOW is unavailable and closes the descriptor', + async () => { + const { slot, metadata } = fixture(); + const outside = fixture(); + vi.doMock('fs', async (original) => { + const actual = await original(); + return { + ...actual, + default: { ...fs, constants: { ...fs.constants, O_NOFOLLOW: undefined } }, + }; + }); + const open = fs.openSync.bind(fs); + vi.spyOn(fs, 'openSync').mockImplementation((file, flags, mode) => { + if (file === metadata) { + fs.unlinkSync(metadata); + fs.symlinkSync(outside.metadata, metadata); + } + return open(file, flags, mode); + }); + const close = vi.spyOn(fs, 'closeSync'); + const read = vi.spyOn(fs, 'readFileSync'); + vi.resetModules(); + const { resolveGraphPath: resolveWithoutNoFollow } = + await import('../../src/storage/shared-store.js'); + expect(resolveWithoutNoFollow(slot)).toBe(path.join(slot, 'lbug')); + expect(read).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + }, +); + +it.each(['../outside', '/outside/lbug', '..\\outside', `../${first}`])( + 'rejects an escaping graph pointer %s without accessing it', + (graphPath) => { + const { slot } = fixture(); + const stat = vi.spyOn(fs, 'lstatSync'); + expect(resolveGraphPath(slot, { graphPath })).toBe(path.join(slot, 'lbug')); + expect(stat).not.toHaveBeenCalled(); + }, +); From a7bb20aea3d856a57dcbcef28b32077d385fd3c4 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 11:44:07 +0000 Subject: [PATCH 33/67] fix(storage): guard the graph path directly Apply the containment check to the same path passed to lstat. The nested normalization guarded a different expression in CodeQL. path.relative already resolves its inputs, so remove the redundant normalization and retain relative-path behavior. --- gitnexus/src/storage/shared-store.ts | 2 +- gitnexus/test/unit/shared-store-read-security.test.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index 8c955ef96..50bc3bd4a 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -296,7 +296,7 @@ export const resolveGraphPath = ( if (typeof recorded !== 'string') return own; if (/^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { const graph = path.join(storagePath, recorded); - const relative = path.relative(path.resolve(storagePath), path.resolve(graph)); + const relative = path.relative(storagePath, graph); if (relative.startsWith('..') || path.isAbsolute(relative)) return own; // Never follow an edited pointer to a symlink or another slot. A missing // committed generation must fail at open, rather than serve an older lbug. diff --git a/gitnexus/test/unit/shared-store-read-security.test.ts b/gitnexus/test/unit/shared-store-read-security.test.ts index a8d814c4e..88ea59614 100644 --- a/gitnexus/test/unit/shared-store-read-security.test.ts +++ b/gitnexus/test/unit/shared-store-read-security.test.ts @@ -57,6 +57,12 @@ it('reads the opened metadata snapshot and invalidates it after pathname replace expect(resolveGraphPath(slot)).toBe(path.join(slot, second)); }); +it('preserves a relative storage path while validating its generation', () => { + const { slot } = fixture(); + const relative = path.relative(process.cwd(), slot); + expect(resolveGraphPath(relative, { graphPath: first })).toBe(path.join(relative, first)); +}); + it.skipIf(process.platform === 'win32').each([false, true])( 'rejects symlinked metadata with a warm cache of %s', (warm) => { From cf3ea83dae569ecea620161d608d6a5be941c5a3 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 12:35:28 +0000 Subject: [PATCH 34/67] fix(embeddings): preserve AST inputs on resume Carry indexed source policy into bounded resumable jobs and preserve exact-input reuse across sibling edits. Include strategy and retention in job identity; admit source memory before reading. --- .../src/core/embeddings/embedding-inputs.ts | 122 ++++++++ .../core/embeddings/embedding-job-identity.ts | 12 +- .../core/embeddings/embedding-job-runner.ts | 287 +++++++++++------- gitnexus/src/core/embeddings/embedding-job.ts | 7 + gitnexus/src/core/embeddings/source-reader.ts | 48 ++- .../src/core/lbug/legacy-vector-source.ts | 63 +--- gitnexus/src/core/lbug/vector-store.ts | 97 ++++-- .../embedding-job-controller.test.ts | 108 +++++++ .../embedding-vector-store.test.ts | 30 ++ .../test/unit/embedding-job-journal.test.ts | 14 + .../test/unit/embedding-source-reader.test.ts | 36 ++- .../unit/embedding-streaming-text.test.ts | 133 +++++++- 12 files changed, 772 insertions(+), 185 deletions(-) create mode 100644 gitnexus/src/core/embeddings/embedding-inputs.ts diff --git a/gitnexus/src/core/embeddings/embedding-inputs.ts b/gitnexus/src/core/embeddings/embedding-inputs.ts new file mode 100644 index 000000000..4aafebac1 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-inputs.ts @@ -0,0 +1,122 @@ +/** Shared streaming input layout for generation and historical-vector validation. */ +import { isNotebookPath } from '../ingestion/ipynb-extractor.js'; +import { ensureAndParse } from './ast-utils.js'; +import { iterateNodeChunks, type EmbeddingChunk } from './chunker.js'; +import { extractStructuralNames } from './structural-extractor.js'; +import { + generateEmbeddingText, + splitEmbeddingText, + withoutOverlappingContext, +} from './text-generator.js'; +import { needsEmbeddingSource, type EmbeddingSourceContext } from './source-reader.js'; +import { + isShortLabel, + STRUCTURAL_LABELS, + type EmbeddableNode, + type EmbeddingConfig, +} from './types.js'; + +export interface EmbeddingInput { + text: string; + startLine: number; + endLine: number; +} + +export async function* iterateEmbeddingInputs( + indexedNode: EmbeddableNode, + config: EmbeddingConfig, + options: { + source?: EmbeddingSourceContext; + readSource?: (node: EmbeddableNode) => Promise<{ content: string; release: () => void }>; + signal?: AbortSignal; + } = {}, +): AsyncGenerator { + const hydrate = + options.source?.contentRetention === 'full' && needsEmbeddingSource(indexedNode, config); + if (hydrate && !options.readSource) + throw new Error('Original embedding source needs a budgeted reader'); + const lease = hydrate ? await options.readSource?.(indexedNode) : undefined; + // Never retain expanded source or structural metadata in the fetched unit. + const node = { ...indexedNode, content: lease?.content ?? indexedNode.content }; + try { + options.signal?.throwIfAborted(); + let classTree: Awaited> | undefined; + if (node.label === 'Class') { + try { + classTree = await ensureAndParse(node.content, node.filePath); + } catch { + classTree = null; + } + } + if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames( + node.content, + node.filePath, + classTree, + node.label === 'Class' ? node.name : undefined, + ); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + /* Structural enrichment remains optional. */ + } + } + const startLine = node.startLine ?? 0, + endLine = node.endLine ?? 0; + const chunks: Iterable | AsyncIterable = isShortLabel( + node.label, + ) + ? [ + { + text: node.content, + chunkIndex: 0, + startLine, + endLine, + startOffset: 0, + endOffset: node.content.length, + }, + ] + : iterateNodeChunks( + node.label, + node.content, + node.filePath, + startLine, + endLine, + config.chunkSize, + config.overlap, + { strategy: config.chunkStrategy, name: node.name, tree: classTree }, + ); + const conservativeRange = + options.source && + (options.source.contentRetention !== 'full' || isNotebookPath(node.filePath)); + let prevTail = '', + previousEndOffset: number | undefined; + for await (const chunk of chunks) { + options.signal?.throwIfAborted(); + node.classDeclaration = chunk.classDeclaration; + const text = generateEmbeddingText( + node, + chunk.text, + config, + chunk.chunkIndex, + withoutOverlappingContext(prevTail, previousEndOffset, chunk.startOffset), + ); + for (const part of splitEmbeddingText(text, config.maxInputChars)) { + // Detach slices so queued inputs cannot pin a large declaration/source. + yield { + text: + lease || text.length > config.maxInputChars + ? Buffer.from(part, 'utf16le').toString('utf16le') + : part, + startLine: conservativeRange ? startLine : chunk.startLine, + endLine: conservativeRange ? endLine : chunk.endLine, + }; + } + prevTail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + previousEndOffset = chunk.endOffset; + } + } finally { + lease?.release(); + } +} diff --git a/gitnexus/src/core/embeddings/embedding-job-identity.ts b/gitnexus/src/core/embeddings/embedding-job-identity.ts index a3c7bec5d..c55d4930d 100644 --- a/gitnexus/src/core/embeddings/embedding-job-identity.ts +++ b/gitnexus/src/core/embeddings/embedding-job-identity.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import type { EmbeddingIdentity } from './embedding-identity.js'; import type { EmbeddingConfig } from './types.js'; +import type { ContentRetention } from '../../storage/repo-meta.js'; export interface EmbeddingGraphIdentity { generation: string; @@ -23,7 +24,9 @@ export function embeddingJobIdentity( config: Pick< EmbeddingConfig, 'chunkSize' | 'overlap' | 'maxSnippetLength' | 'maxDescriptionLength' | 'maxInputChars' - >, + > & + Partial>, + contentRetention: ContentRetention = 'full', ): { identity: string; sourceIdentity: string; vectorIdentity: string; inputIdentity: string } { const source = [ graph.generation, @@ -50,13 +53,18 @@ export function embeddingJobIdentity( ].every((n) => Number.isSafeInteger(n) && n >= 0) || config.chunkSize < 1 || config.overlap >= config.chunkSize || - config.maxInputChars < 2 + config.maxInputChars < 2 || + !['ast', 'fixed'].includes(config.chunkStrategy ?? 'ast') || + !['full', 'symbol', 'none'].includes(contentRetention) ) throw new Error('Incomplete embedding job identity'); const sourceIdentity = digest(source); const vectorIdentity = digest([vector.provider, vector.model, vector.dimensions]); const inputIdentity = digest([ + 'source-aware-ast-inputs:v1', textVersion, + config.chunkStrategy ?? 'ast', + contentRetention, config.chunkSize, config.overlap, config.maxSnippetLength, diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index 19b8ae56b..aa4d1fff9 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -1,4 +1,5 @@ /** One durable manifest unit, one bounded provider batch, no corpus-sized maps. */ +import { createHash } from 'node:crypto'; import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; import type { VectorNodeReceipt, VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; import type { @@ -14,16 +15,14 @@ import { type EmbeddingWorkUnit, } from './embedding-work-manifest.js'; import { contentHashForNode, embedInputRanges, isInputRejection } from './embedding-pipeline.js'; -import { iterateNodeChunks } from './chunker.js'; -import { generateEmbeddingText, splitEmbeddingText } from './text-generator.js'; -import { extractStructuralNames } from './structural-extractor.js'; -import { resolveEmbeddingConfig } from './config.js'; +import { iterateEmbeddingInputs, type EmbeddingInput } from './embedding-inputs.js'; import { - isShortLabel, - STRUCTURAL_LABELS, - type EmbeddingConfig, - type EmbeddableNode, -} from './types.js'; + EmbeddingSourceReader, + needsEmbeddingSource, + type EmbeddingSourceContext, +} from './source-reader.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { type EmbeddingConfig, type EmbeddableNode } from './types.js'; import type { embedBatch } from './embedder.js'; export interface EmbeddingReuseSource { @@ -34,6 +33,8 @@ export interface EmbeddingReuseSource { contentHash: string, node: EmbeddableNode, prepareInputValidation?: () => Promise, + inputs?: () => AsyncIterable, + matchesHash?: (stored: string) => Promise, ): Promise; readChunks: VectorStore['readChunks']; } @@ -42,6 +43,7 @@ export interface EmbeddingJobRunnerOptions { directory: string; manifest: EmbeddingManifestReceipt; source: EmbeddingManifestSource; + sourceContext?: EmbeddingSourceContext; journal: EmbeddingJobJournal; store: VectorStore; /** The controller has validated vector/input identities before providing reuse. */ @@ -71,6 +73,8 @@ export async function runEmbeddingJob( const { directory, manifest, source, journal, store, signal } = options; const config = resolveEmbeddingConfig(options.config ?? {}); const budget = new EmbeddingByteBudget(options.queueBytes); + let sourceReader: EmbeddingSourceReader | undefined; + let releaseSourceCache: (() => void) | undefined; const dimensions = store.options.dimensions; // UTF-16 text + worst-case JSON escaping/encoding, bounded HTTP response // decoding/JSON parsing, float results, JS arrays and native copies. HTTP @@ -119,7 +123,14 @@ export async function runEmbeddingJob( Math.floor(store.options.batchBytes / (16 * dimensions + 128 + 2 * (maxIdChars + 40))), ); if (maximumBatchRows < 1) throw new Error('Vector write does not fit its batch reservation'); - if (retainedBytes + scratchBytes + rowBytes(config.maxInputChars, maxIdChars) > budget.limit) + const minimumUnitBatchBytes = rowBytes(config.maxInputChars, maxIdChars); + const unitBytes = retainedBytes + scratchBytes + minimumUnitBatchBytes; + if (releaseSourceCache && budget.usedBytes + unitBytes > budget.limit) { + sourceReader = undefined; + releaseSourceCache(); + releaseSourceCache = undefined; + } + if (budget.usedBytes + unitBytes > budget.limit) throw new Error( `Embedding unit ${unit.ordinal} requires a larger resource budget; no source text was truncated`, ); @@ -145,7 +156,10 @@ export async function runEmbeddingJob( let copyBatch: VectorWriteChunk[] = []; let copyBytes = 0; let copyReleases: Array<() => void> = []; - const generated = new Map(); + const generated = new Map< + string, + { contentHash: string; chunks: number; ended: boolean; inputsDigest?: string } + >(); const failed = new Set(); let failureCause: 'input' | 'provider' = 'input'; let abort: unknown; @@ -243,22 +257,108 @@ export async function runEmbeddingJob( } : undefined; + const nodeInputs = (node: EmbeddableNode) => + iterateEmbeddingInputs(node, config, { + source: options.sourceContext, + signal, + readSource: async (indexedNode) => { + const sourceContext = options.sourceContext; + if (!sourceContext) throw new Error('Missing indexed embedding source context'); + let release: (() => void) | undefined; + try { + if (!sourceReader) { + const cacheBytes = Math.min(16 * 1024 * 1024, Math.floor(budget.limit / 8)); + if (budget.usedBytes + cacheBytes + minimumUnitBatchBytes > budget.limit) { + await flush(); + await flushCopies(); + } + if (abort) throw abort; + releaseSourceCache = budget.reserve(cacheBytes); + sourceReader = new EmbeddingSourceReader(sourceContext, cacheBytes); + } + const content = await sourceReader.read(indexedNode, async (fileBytes) => { + signal?.throwIfAborted(); + // Whole-file decoding/line index, notebook reconstruction, AST and + // projected declarations live until this node's iterator closes. + const bytes = 64 * 1024 + 16 * fileBytes; + if (budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit) { + await flush(); + await flushCopies(); + } + if (abort) throw abort; + if ( + !Number.isSafeInteger(bytes) || + budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit + ) + throw new Error( + 'Original embedding source requires a larger resource budget; no source text was truncated', + ); + release = budget.reserve(bytes); + }); + if (!release) throw new Error('Original embedding source was not admitted'); + return { content, release }; + } catch (error) { + release?.(); + throw error; + } + }, + }); + + const sourceAware = (node: EmbeddableNode) => + options.sourceContext?.contentRetention === 'full' && needsEmbeddingSource(node, config); + const inputHasher = () => + createHash('sha1').update('embedding-inputs:v1\n').update(JSON.stringify(config.modelId)); + try { // Hash once per fetched node; current-job complete groups never fetch/hash. const inputs = fetched.nodes.map((node) => ({ nodeId: node.id, - contentHash: contentHashForNode(node, config), + contentHash: contentHashForNode(node, config, options.sourceContext), sourceStartLine: node.startLine ?? 0, sourceEndLine: node.endLine ?? 0, })); - if (options.revalidateStoredInputs) { - for (const input of inputs) { - const retained = await store.reuseNode(input); + const reusableNodes = new Map(); + for (let i = 0; i < inputs.length; i++) { + const input = inputs[i], + node = fetched.nodes[i]; + const baseHash = input.contentHash; + let digest: string | undefined; + const matchesHash = async (previous: string): Promise => { + if (!sourceAware(node) || !/^[a-f0-9]{40}:[a-f0-9]{40}$/.test(previous)) return false; + if (previous.startsWith(`${baseHash}:`)) { + input.contentHash = previous; + return true; + } + if (!digest) { + const hash = inputHasher(); + for await (const part of nodeInputs(node)) + hash.update(JSON.stringify([part.text, part.startLine, part.endLine])); + digest = hash.digest('hex'); + } + if (!previous.endsWith(`:${digest}`)) return false; + input.contentHash = `${baseHash}:${digest}`; + return true; + }; + if (options.revalidateStoredInputs) { + const retained = await store.reuseNode(input, async (previous) => + (await matchesHash(previous)) ? input.contentHash : undefined, + ); if (retained) { complete.set(input.nodeId, retained); reusedNodes++; + continue; } } + const reusable = await options.reuse?.reusable( + node.id, + input.contentHash, + node, + prepareInputValidation, + () => nodeInputs(node), + matchesHash, + ); + if (reusable) reusableNodes.set(node.id, reusable); + else input.contentHash = baseHash; } const missingInputs = options.revalidateStoredInputs ? inputs.filter((input) => !complete.has(input.nodeId)) @@ -271,12 +371,7 @@ export async function runEmbeddingJob( signal?.throwIfAborted(); if (options.revalidateStoredInputs && complete.has(node.id)) continue; const reuse = options.reuse; - const reusable = await reuse?.reusable( - node.id, - input.contentHash, - node, - prepareInputValidation, - ); + const reusable = reusableNodes.get(node.id); if (reusable && reuse) { // Flush before copy so there is only one vector batch in memory. await flush(); @@ -337,76 +432,51 @@ export async function runEmbeddingJob( continue; } if (copyBatch.length) await flushCopies(); - if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { - try { - const names = await extractStructuralNames(node.content, node.filePath); - node.methodNames = names.methodNames; - node.fieldNames = names.fieldNames; - } catch { - /* Same optional structural enrichment as the legacy pipeline. */ - } - } - const state = { contentHash: input.contentHash, chunks: 0, ended: false }; + const state = { + contentHash: input.contentHash, + chunks: 0, + ended: false, + inputsDigest: undefined as string | undefined, + }; generated.set(node.id, state); - const chunks = isShortLabel(node.label) - ? [ - { - text: node.content, - chunkIndex: 0, - startLine: node.startLine ?? 0, - endLine: node.endLine ?? 0, - }, - ] - : iterateNodeChunks( - node.label, - node.content, - node.filePath, - node.startLine ?? 0, - node.endLine ?? 0, - config.chunkSize, - config.overlap, - ); - let prevTail = ''; - for await (const chunk of chunks) { + const hash = sourceAware(node) ? inputHasher() : undefined; + for await (const generatedInput of nodeInputs(node)) { if (failed.has(node.id) || abort) break; - const text = generateEmbeddingText(node, chunk.text, config, chunk.chunkIndex, prevTail); - for (const inputText of splitEmbeddingText(text, config.maxInputChars)) { - const bytes = rowBytes(inputText.length, node.id.length); - if ( - batch.length && - (batch.length >= maximumBatchRows || budget.usedBytes + bytes > budget.limit) - ) - await flush(); - if (failed.has(node.id) || abort) break; - const release = budget.reserve(bytes); - // A sliced string can pin an entire oversized generated declaration. - // UTF-16 detachment preserves even unpaired surrogate code units. - const detached = - text.length > config.maxInputChars - ? Buffer.from(inputText, 'utf16le').toString('utf16le') - : inputText; - batch.push({ - ...input, - chunkIndex: state.chunks++, - startLine: chunk.startLine, - endLine: chunk.endLine, - text: detached, - release, - }); - } - prevTail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + hash?.update( + JSON.stringify([generatedInput.text, generatedInput.startLine, generatedInput.endLine]), + ); + const bytes = rowBytes(generatedInput.text.length, node.id.length); + if ( + batch.length && + (batch.length >= maximumBatchRows || budget.usedBytes + bytes > budget.limit) + ) + await flush(); + if (failed.has(node.id) || abort) break; + const release = budget.reserve(bytes); + batch.push({ + ...input, + chunkIndex: state.chunks++, + ...generatedInput, + release, + }); } state.ended = !failed.has(node.id) && !abort; - node.methodNames = undefined; - node.fieldNames = undefined; + if (state.ended) state.inputsDigest = hash?.digest('hex'); if (abort) break; } if (copyBatch.length) await flushCopies(); if (!abort) await flush(); - const finished = [...generated] - .filter(([id, node]) => node.ended && !failed.has(id)) - .map(([nodeId, node]) => ({ nodeId, contentHash: node.contentHash, chunks: node.chunks })) - .concat(copied); + const finished = [ + ...[...generated] + .filter(([id, node]) => node.ended && !failed.has(id)) + .map(([nodeId, node]) => ({ + nodeId, + contentHash: node.contentHash, + chunks: node.chunks, + inputsDigest: node.inputsDigest, + })), + ...copied, + ]; if (finished.length) for (const receipt of await store.finishNodes(finished)) complete.set(receipt.nodeId, receipt); @@ -447,29 +517,34 @@ export async function runEmbeddingJob( } }; - if (journal.snapshot().phase === 'embedding') { - const active = journal.snapshot().active; - if (active) await replay(active); - for await (const intent of journal.failed()) await replay(intent); - let cursor = journal.snapshot().cursor; - for await (const { unit, next } of readEmbeddingWorkUnits(directory, manifest, cursor)) { - await processUnit(unit, { - start: cursor, - next, - nodes: unit.nodes.length, - origin: 'manifest', - }); - cursor = next; + try { + if (journal.snapshot().phase === 'embedding') { + const active = journal.snapshot().active; + if (active) await replay(active); + for await (const intent of journal.failed()) await replay(intent); + let cursor = journal.snapshot().cursor; + for await (const { unit, next } of readEmbeddingWorkUnits(directory, manifest, cursor)) { + await processUnit(unit, { + start: cursor, + next, + nodes: unit.nodes.length, + origin: 'manifest', + }); + cursor = next; + } + if (journal.snapshot().failedUnits) + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes }; + await journal.indexing(); + progress(); } - if (journal.snapshot().failedUnits) - return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes }; - await journal.indexing(); - progress(); + signal?.throwIfAborted(); + if (options.revalidateStoredInputs && !store.isSealed) + await store.pruneUnselected(journal.snapshot().identity, manifest.totalNodes); + signal?.throwIfAborted(); + const indexKind = await store.seal(manifest.totalNodes, journal.snapshot().chunks); + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; + } finally { + sourceReader = undefined; + releaseSourceCache?.(); } - signal?.throwIfAborted(); - if (options.revalidateStoredInputs && !store.isSealed) - await store.pruneUnselected(journal.snapshot().identity, manifest.totalNodes); - signal?.throwIfAborted(); - const indexKind = await store.seal(manifest.totalNodes, journal.snapshot().chunks); - return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; } diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 72aa49c9a..e44846826 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -19,6 +19,7 @@ import { metadataMatchesGraph, } from '../../storage/vector-generation.js'; import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; +import { contentRetentionFromMeta } from '../content-retention.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; import { LegacyVectorSource, legacyVectorCompatibility } from '../lbug/legacy-vector-source.js'; @@ -197,6 +198,7 @@ export async function runPublishedEmbeddingJob( vector, EMBEDDING_TEXT_VERSION, config, + contentRetentionFromMeta(metadata), ); const previous = metadata.vectorGeneration; if (previous?.identity === identities.identity && !metadata.embeddingJob) { @@ -494,6 +496,11 @@ export async function runPublishedEmbeddingJob( revalidateStoredInputs: owner.revalidateInputs && !store.isSealed, config, queueBytes: resources.queueBytes, + sourceContext: { + repoPath: metadata.repoPath, + fileHashes: metadata.fileHashes, + contentRetention: contentRetentionFromMeta(metadata), + }, // Opening the graph is lazy even on resume: indexing-only retries do not // enumerate/fetch source or initialize a provider. source: { diff --git a/gitnexus/src/core/embeddings/source-reader.ts b/gitnexus/src/core/embeddings/source-reader.ts index 26344214f..4b71569f7 100644 --- a/gitnexus/src/core/embeddings/source-reader.ts +++ b/gitnexus/src/core/embeddings/source-reader.ts @@ -1,7 +1,8 @@ import { createHash } from 'node:crypto'; -import { readFile, realpath } from 'node:fs/promises'; +import { realpath } from 'node:fs/promises'; import path from 'node:path'; import type { ContentRetention } from '../../storage/repo-meta.js'; +import { openEmbeddingJobFile } from '../../storage/embedding-job.js'; import { isNotebookPath, notebookPythonSnippet } from '../ingestion/ipynb-extractor.js'; import { buildLineIndex } from './line-index.js'; import { @@ -47,6 +48,7 @@ interface SourceFile { content: string; lines: Int32Array; bytes: number; + sourceBytes: number; } const CACHE_BYTES = 16 * 1024 * 1024; @@ -58,9 +60,17 @@ export class EmbeddingSourceReader { private bytes = 0; private root: Promise | undefined; - constructor(private readonly source: EmbeddingSourceContext) {} + constructor( + private readonly source: EmbeddingSourceContext, + private readonly cacheBytes = CACHE_BYTES, + ) {} - async read(node: EmbeddableNode): Promise { + /** Admission runs before allocation, including on cache hits. The caller + * holds its reservation until it finishes using/parsing the returned text. */ + async read( + node: EmbeddableNode, + admit?: (sourceBytes: number) => Promise, + ): Promise { if (this.source.contentRetention !== 'full') { throw new Error('Original embedding source requires full content retention'); } @@ -84,6 +94,7 @@ export class EmbeddingSourceReader { let file = this.files.get(node.filePath); if (file) { + await admit?.(file.sourceBytes); this.files.delete(node.filePath); this.files.set(node.filePath, file); } else { @@ -107,15 +118,38 @@ export class EmbeddingSourceReader { ) { throw new Error('source path leaves the repository'); } - const buffer = await readFile(resolved); + const { handle, stat } = await openEmbeddingJobFile(resolved); + let buffer: Buffer; + try { + if (!Number.isSafeInteger(stat.size) || stat.size < 0) + throw new Error('invalid source file size'); + await admit?.(stat.size); + // Bound the read to the admitted size, plus one byte to detect growth. + buffer = Buffer.alloc(stat.size + 1); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset); + if (!bytesRead) break; + offset += bytesRead; + } + if (offset !== stat.size) throw new Error('source size changed during the read'); + buffer = buffer.subarray(0, offset); + } finally { + await handle.close(); + } if (createHash('sha256').update(buffer).digest('hex') !== expectedHash) { throw new Error('checkout content differs from the indexed snapshot'); } const content = buffer.toString('utf8'); const lines = buildLineIndex(content); - file = { content, lines, bytes: content.length * 2 + lines.byteLength }; - if (file.bytes <= CACHE_BYTES) { - while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > CACHE_BYTES) { + file = { + content, + lines, + bytes: content.length * 2 + lines.byteLength, + sourceBytes: buffer.length, + }; + if (file.bytes <= this.cacheBytes) { + while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > this.cacheBytes) { const entry = this.files.entries().next().value; if (!entry) break; const [oldest, cached] = entry; diff --git a/gitnexus/src/core/lbug/legacy-vector-source.ts b/gitnexus/src/core/lbug/legacy-vector-source.ts index 2978b4aea..e9688d637 100644 --- a/gitnexus/src/core/lbug/legacy-vector-source.ts +++ b/gitnexus/src/core/lbug/legacy-vector-source.ts @@ -15,15 +15,8 @@ import { import { assertValidGraphNodeId } from '../graph/node-identity.js'; import type { EmbeddingIdentity } from '../embeddings/embedding-identity.js'; import { resolveEmbeddingConfig } from '../embeddings/config.js'; -import { iterateNodeChunks } from '../embeddings/chunker.js'; -import { generateEmbeddingText, splitEmbeddingText } from '../embeddings/text-generator.js'; -import { extractStructuralNames } from '../embeddings/structural-extractor.js'; -import { - isShortLabel, - STRUCTURAL_LABELS, - type EmbeddableNode, - type EmbeddingConfig, -} from '../embeddings/types.js'; +import { iterateEmbeddingInputs, type EmbeddingInput } from '../embeddings/embedding-inputs.js'; +import { isShortLabel, type EmbeddableNode, type EmbeddingConfig } from '../embeddings/types.js'; export interface LegacyVectorSourceOptions { /** Caller validates ownership and holds the slot lock. WAL recovery and @@ -266,14 +259,24 @@ export class LegacyVectorSource { contentHash: string, node: EmbeddableNode, prepareInputValidation?: () => Promise, + inputs?: () => AsyncIterable, + matchesHash?: (stored: string) => Promise, ): Promise { this.acquire(); const originalMethods = node.methodNames, originalFields = node.fieldNames; try { const group = await this.index.legacyReuseGroup(this.options.sourceIdentity, nodeId); - if (!group || group.contentHash !== contentHash || node.id !== nodeId) return null; - if (!this.options.recordedInputIdentity && !isShortLabel(node.label) && group.chunks > 1) { + if (!group || node.id !== nodeId) return null; + const sourceAware = /^[a-f0-9]{40}:[a-f0-9]{40}$/.test(group.contentHash); + if (group.contentHash !== contentHash && !(await matchesHash?.(group.contentHash))) + return null; + if ( + !this.options.recordedInputIdentity && + !sourceAware && + !isShortLabel(node.label) && + group.chunks > 1 + ) { return null; } await prepareInputValidation?.(); @@ -288,7 +291,7 @@ export class LegacyVectorSource { let block: Row[] = [], ordinal = 0, delta: number | undefined; - for await (const span of this.inputSpans(node)) { + for await (const span of inputs ? inputs() : this.inputSpans(node)) { if (ordinal >= group.chunks) return null; if (ordinal % blockSize === 0) block = await this.rows( @@ -323,41 +326,7 @@ export class LegacyVectorSource { private async *inputSpans( node: EmbeddableNode, ): AsyncGenerator<{ startLine: number; endLine: number }> { - if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { - try { - const names = await extractStructuralNames(node.content, node.filePath); - node.methodNames = names.methodNames; - node.fieldNames = names.fieldNames; - } catch { - /* Same optional enrichment and text-version contract as generation. */ - } - } - const chunks = isShortLabel(node.label) - ? [ - { - text: node.content, - chunkIndex: 0, - startLine: node.startLine ?? 0, - endLine: node.endLine ?? 0, - }, - ] - : iterateNodeChunks( - node.label, - node.content, - node.filePath, - node.startLine ?? 0, - node.endLine ?? 0, - this.config.chunkSize, - this.config.overlap, - ); - let prevTail = ''; - for await (const chunk of chunks) { - this.options.signal?.throwIfAborted(); - const text = generateEmbeddingText(node, chunk.text, this.config, chunk.chunkIndex, prevTail); - for (const _part of splitEmbeddingText(text, this.config.maxInputChars)) - yield { startLine: chunk.startLine, endLine: chunk.endLine }; - prevTail = this.config.overlap > 0 ? chunk.text.slice(-this.config.overlap) : ''; - } + yield* iterateEmbeddingInputs(node, this.config, { signal: this.options.signal }); } async readChunks( diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 467f5ae65..0b04dbcf0 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -265,8 +265,13 @@ export class VectorStore { /** Reuse a complete group in the pending writer after validating its CURRENT * input. Relocation updates only line metadata, with no vector round trip. - * One transaction makes an interrupted relocation safe to retry. */ - async reuseNode(input: VectorNodeInput): Promise { + * One transaction makes an interrupted relocation safe to retry. + * resolveHash permits a caller-verified exact-input match after the source + * snapshot changed. Verification and hash/range replacement are atomic. */ + async reuseNode( + input: VectorNodeInput, + resolveHash?: (stored: string) => Promise, + ): Promise { assertValidGraphNodeId(input.nodeId, 'Vector store'); const startLine = input.sourceStartLine ?? 0; const endLine = input.sourceEndLine ?? 0; @@ -279,10 +284,18 @@ export class VectorStore { throw new Error('Invalid reusable embedding input'); return this.exclusive(async () => { const previous = await this.receipt(input.nodeId); - if (!previous?.complete || previous.chunks < 1 || previous.contentHash !== input.contentHash) - return null; + if (!previous?.complete || previous.chunks < 1) return null; + const contentHash = + previous.contentHash === input.contentHash + ? input.contentHash + : await resolveHash?.(previous.contentHash); + if (!contentHash) return null; await this.verify(previous); - if (previous.sourceStartLine === startLine && previous.sourceEndLine === endLine) + if ( + previous.sourceStartLine === startLine && + previous.sourceEndLine === endLine && + previous.contentHash === contentHash + ) return previous; const delta = startLine - (previous.sourceStartLine ?? 0); await this.transaction(async () => { @@ -292,8 +305,14 @@ export class VectorStore { (_, i) => `${input.nodeId}:${start + i}`, ); await this.query( - 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.startLine=e.startLine+$delta, e.endLine=CASE WHEN e.endLine=$oldEnd THEN $endLine ELSE e.endLine+$delta END', - { ids, delta, oldEnd: previous.sourceEndLine ?? 0, endLine }, + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.contentHash=$contentHash, e.startLine=e.startLine+$delta, e.endLine=CASE WHEN e.endLine=$oldEnd THEN $endLine ELSE e.endLine+$delta END', + { + ids, + delta, + oldEnd: previous.sourceEndLine ?? 0, + endLine, + contentHash, + }, ); const invalid = await this.query( 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) WHERE e.startLine < 0 OR e.endLine < e.startLine RETURN count(e) AS invalid', @@ -302,11 +321,16 @@ export class VectorStore { if (count(invalid[0].invalid)) throw new Error('Invalid relocated vector lines'); } await this.query( - 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.sourceStartLine=$startLine, n.sourceEndLine=$endLine', - { nodeId: input.nodeId, startLine, endLine }, + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.sourceStartLine=$startLine, n.sourceEndLine=$endLine', + { nodeId: input.nodeId, startLine, endLine, contentHash }, ); }); - return { ...previous, sourceStartLine: startLine, sourceEndLine: endLine }; + return { + ...previous, + contentHash, + sourceStartLine: startLine, + sourceEndLine: endLine, + }; }, true); } @@ -547,7 +571,7 @@ export class VectorStore { } async finishNodes( - nodes: readonly (VectorNodeInput & { chunks: number })[], + nodes: readonly (VectorNodeInput & { chunks: number; inputsDigest?: string })[], ): Promise { if ( !nodes.length || @@ -564,24 +588,61 @@ export class VectorStore { !receipt || receipt.contentHash !== node.contentHash || receipt.chunks !== node.chunks || - node.chunks < 1 + node.chunks < 1 || + (node.inputsDigest !== undefined && + (!/^[a-f0-9]{40}$/.test(node.inputsDigest) || !/^[a-f0-9]{40}$/.test(node.contentHash))) ) throw new Error('Vector node coverage does not match its generated chunks'); verified.push(receipt); } await this.verify(verified); - await this.query( - 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', - { ids: nodes.map((n) => n.nodeId) }, - ); + const finalize = async () => { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i]; + if (!node.inputsDigest) continue; + const contentHash = `${node.contentHash}:${node.inputsDigest}`; + for (let start = 0; start < node.chunks; start += BLOCK_ROWS) { + const ids = Array.from( + { length: Math.min(BLOCK_ROWS, node.chunks - start) }, + (_, j) => `${node.nodeId}:${start + j}`, + ); + await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.contentHash=$contentHash', + { ids, contentHash }, + ); + } + await this.query( + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash', + { nodeId: node.nodeId, contentHash }, + ); + verified[i] = { ...verified[i], contentHash }; + } + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', + { ids: nodes.map((n) => n.nodeId) }, + ); + }; + if (nodes.some((node) => node.inputsDigest)) await this.transaction(finalize); + else await finalize(); return verified.map((receipt) => ({ ...receipt, complete: true })); }, true); } - async reusable(nodeId: string, contentHash: string): Promise { + async reusable( + nodeId: string, + contentHash: string, + _node?: unknown, + _prepare?: unknown, + _inputs?: unknown, + matchesHash?: (stored: string) => Promise, + ): Promise { return this.exclusive(async () => { const receipt = await this.receipt(nodeId); - if (!receipt?.complete || receipt.contentHash !== contentHash || receipt.chunks < 1) + if ( + !receipt?.complete || + receipt.chunks < 1 || + (receipt.contentHash !== contentHash && !(await matchesHash?.(receipt.contentHash))) + ) return null; await this.verify(receipt); return receipt; diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index 067490107..aa98c6964 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -12,6 +12,8 @@ import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js' import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { EmbeddingSourceReader } from '../../src/core/embeddings/source-reader.js'; let temporary: TestDBHandle | undefined; afterEach(async () => { @@ -138,6 +140,112 @@ async function fixture(count = 130) { } describe('pending artifacts across graph generations', () => { + it.each(['full', 'symbol', 'none'] as const)( + 'uses published repository source policy (%s), then resumes without source or provider work', + async (retention) => { + const f = await fixture(1); + const repo = path.join(f.root, 'checkout'); + await fs.mkdir(repo); + const content = `function sample() {\n${' consume("padding");\n'.repeat(300)} return "RECOVERED_TAIL";\n}`; + await fs.writeFile(path.join(repo, 'sample.ts'), content); + const node: EmbeddableNode = { + id: 'Function:sample.ts:sample', + label: 'Function', + name: 'sample', + filePath: 'sample.ts', + content: content.slice(0, 5000) + '... [truncated]', + startLine: 0, + endLine: content.split('\n').length - 1, + }; + f.scan.mockImplementation(async function* (label) { + if (label === 'Function') + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }); + f.fetch.mockImplementation(async () => [node]); + const meta = await loadMeta(f.root); + if (!meta) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...meta, + repoPath: repo, + contentRetention: retention, + fileHashes: { 'sample.ts': createHash('sha256').update(content).digest('hex') }, + }); + const read = vi.spyOn(EmbeddingSourceReader.prototype, 'read'); + const result = await f.run(); + const texts = f.provider.mock.calls.flatMap(([texts]) => texts).join('\n'); + expect(texts.includes('RECOVERED_TAIL')).toBe(retention === 'full'); + expect(read.mock.calls.length > 0).toBe(retention === 'full'); + expect(result.peakQueueBytes).toBeLessThanOrEqual(f.options().resources.queueBytes); + expect(node.content).toContain('[truncated]'); + if (retention === 'full') { + const advance = async (source: string) => { + await fs.writeFile(path.join(repo, 'sample.ts'), source); + await f.nextGeneration(); + const current = await loadMeta(f.root); + if (!current) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...current, + repoPath: repo, + contentRetention: retention, + fileHashes: { 'sample.ts': createHash('sha256').update(source).digest('hex') }, + }); + }; + f.provider.mockClear(); + read.mockClear(); + await advance(content + '\n// unrelated sibling edit'); + expect((await f.run()).state?.reusedNodes).toBe(1); + expect(f.provider).not.toHaveBeenCalled(); + expect(read).toHaveBeenCalled(); + await advance(content.replace('RECOVERED_TAIL', 'CHANGED_TAIL')); + await f.run(); + expect(f.provider.mock.calls.flatMap(([texts]) => texts).join('\n')).toContain( + 'CHANGED_TAIL', + ); + } + await fs.rm(repo, { recursive: true }); + f.provider.mockClear(); + read.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + }, + ); + + it('rejects oversized original source before paid work and retries with a larger reservation', async () => { + const f = await fixture(1); + const content = `function sample() {\n${' '.repeat(1024 * 1024)}return "TAIL";\n}`; + await fs.writeFile(path.join(f.root, 'sample.ts'), content); + const node: EmbeddableNode = { + id: 'Function:sample.ts:sample', + label: 'Function', + name: 'sample', + filePath: 'sample.ts', + content: 'function sample() ... [truncated]', + startLine: 0, + endLine: 2, + }; + f.scan.mockImplementation(async function* (label) { + if (label === 'Function') + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }); + f.fetch.mockImplementation(async () => [node]); + const meta = await loadMeta(f.root); + if (!meta) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...meta, + fileHashes: { 'sample.ts': createHash('sha256').update(content).digest('hex') }, + }); + await expect(f.run()).rejects.toThrow('larger resource budget'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + const result = await f.run({ + resources: { ...f.options().resources, queueBytes: 32 * 1024 * 1024 }, + }); + expect(result.phase).toBe('ready'); + expect(result.peakQueueBytes).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(f.provider.mock.calls.flatMap(([texts]) => texts).join('\n')).toContain('TAIL'); + }); + it('recovers native sealing before the ownership phase was recorded, then reuses read-only', async () => { const f = await fixture(5); const ownership = await import('../../src/storage/embedding-artifact.js'); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 0cd91719c..ae23457ba 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -762,6 +762,36 @@ describe('bounded durable embedding runner', () => { }); describe('separate durable vector artifacts', () => { + it('publishes exact-input digests with rows and updates a verified source hash without replacing vectors', async () => { + const options = await setup(); + const nodeId = 'Function:source.ts:sample', + hash = 'a'.repeat(40), + digest = 'b'.repeat(40); + await store.beginNodes([{ nodeId, contentHash: hash, sourceStartLine: 1, sourceEndLine: 2 }]); + await store.appendBatch([{ nodeId, contentHash: hash, ...chunk(0) }]); + const [receipt] = await store.finishNodes([ + { nodeId, contentHash: hash, chunks: 1, inputsDigest: digest }, + ]); + expect(receipt.contentHash).toBe(`${hash}:${digest}`); + const original = await store.readChunks(receipt, 0, 1); + const next = { + nodeId, + contentHash: `${'c'.repeat(40)}:${digest}`, + sourceStartLine: 1, + sourceEndLine: 2, + }; + expect(await store.reuseNode(next, async () => undefined)).toBeNull(); + const reused = await store.reuseNode(next, async (previous) => + previous === receipt.contentHash ? next.contentHash : undefined, + ); + expect(reused?.contentHash).toBe(next.contentHash); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + const recovered = (await store.completedNodes([nodeId])).get(nodeId); + if (!recovered) throw new Error('Missing completed input receipt'); + expect(recovered.contentHash).toBe(next.contentHash); + expect(await store.readChunks(recovered, 0, 1)).toEqual(original); + }); it('verifies a complete unit with bounded native queries instead of one query per node', async () => { await setup(); const nodes = Array.from({ length: 64 }, (_, i) => ({ diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index 11d0e66d3..53a36d9b7 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -330,4 +330,18 @@ describe('embedding job identity', () => { ), ).toThrow(); }); + it('separates AST/fixed layout and indexed retention without splitting equivalent defaults', () => { + const base = embeddingJobIdentity(graph, vector, 'v7', config); + expect(embeddingJobIdentity(graph, vector, 'v7', { ...config, chunkStrategy: 'ast' })).toEqual( + base, + ); + expect( + embeddingJobIdentity(graph, vector, 'v7', { ...config, chunkStrategy: 'fixed' }) + .inputIdentity, + ).not.toBe(base.inputIdentity); + for (const retention of ['symbol', 'none'] as const) + expect(embeddingJobIdentity(graph, vector, 'v7', config, retention).inputIdentity).not.toBe( + base.inputIdentity, + ); + }); }); diff --git a/gitnexus/test/unit/embedding-source-reader.test.ts b/gitnexus/test/unit/embedding-source-reader.test.ts index 4531d661f..6b308c98f 100644 --- a/gitnexus/test/unit/embedding-source-reader.test.ts +++ b/gitnexus/test/unit/embedding-source-reader.test.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { mkdtemp, mkdir, writeFile, rm, symlink, unlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { EmbeddingSourceReader, needsEmbeddingSource, @@ -38,6 +38,40 @@ async function file(name: string, text: string) { } describe('original embedding source', () => { + it('admits source bytes before reading, including cached files', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source); + const admit = vi.fn(async (): Promise => { + throw new Error('budget exhausted'); + }); + await expect(reader.read(node('sample.py', 0, 0), admit)).rejects.toThrow('budget exhausted'); + expect(admit).toHaveBeenCalledWith(6); + admit.mockImplementation(async () => {}); + expect(await reader.read(node('sample.py', 0, 0), admit)).toBe('source'); + await unlink(path.join(root, 'sample.py')); + expect(await reader.read(node('sample.py', 0, 0), admit)).toBe('source'); + expect(admit).toHaveBeenCalledTimes(3); + }); + + it('rejects growth after admission instead of reading beyond the reservation', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source); + await expect( + reader.read(node('sample.py', 0, 0), async () => { + await writeFile(path.join(root, 'sample.py'), 'source grew'); + }), + ).rejects.toThrow('source size changed'); + }); + + it('honors the job-specific cache allowance', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source, 8); + expect(await reader.read(node('sample.py', 0, 0))).toBe('source'); + await unlink(path.join(root, 'sample.py')); + await expect(reader.read(node('sample.py', 0, 0))).rejects.toThrow( + 'Embedding source unavailable', + ); + }); it('retains indentation, Unicode and exact zero-based inclusive CRLF lines', async () => { const text = '# prefix\r\ndef sample():\r\n value = "雪😀"\r\n return value\r\n# suffix'; await file('sample.py', text); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts index 2b513ce1b..d7c366c15 100644 --- a/gitnexus/test/unit/embedding-streaming-text.test.ts +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -1,8 +1,11 @@ -import { describe, expect, it } from 'vitest'; +import { createHash } from 'node:crypto'; +import { describe, expect, it, vi } from 'vitest'; import { iterateCharacterChunks } from '../../src/core/embeddings/character-chunk.js'; import { buildLineIndex } from '../../src/core/embeddings/line-index.js'; import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; import { extractDeclarationOnly } from '../../src/core/embeddings/text-generator.js'; +import { iterateEmbeddingInputs } from '../../src/core/embeddings/embedding-inputs.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; const fingerprintNode = (label: string, content: string) => ({ id: 'Function:src/example.ts:test', @@ -16,6 +19,127 @@ const fingerprintNode = (label: string, content: string) => ({ }); describe('bounded embedding text preserves existing vectors', () => { + // Exact text AND line-range receipts captured from AST base 6895f91726a3. + it.each([ + [ + 'Function', + 'sample.ts', + 'ast', + 'full', + 12, + 'e3728ed50afd80c3d9da4ba1b2d23c3f906b230851261462e0e554623196f457', + ], + [ + 'Function', + 'sample.ts', + 'fixed', + 'full', + 15, + '718690efd94d6ec0ff458b80280751b16aa57ad285ea5541bbe4f0128ab0999d', + ], + [ + 'Class', + 'sample.ts', + 'ast', + 'full', + 1, + '3af1649850519626a030297788391a745ee4370269f1f44a33f4362f7f38f5c2', + ], + [ + 'Class', + 'sample.ts', + 'fixed', + 'full', + 1, + '3af1649850519626a030297788391a745ee4370269f1f44a33f4362f7f38f5c2', + ], + [ + 'Function', + 'sample.ts', + 'ast', + 'symbol', + 12, + '8c5780484a0f3cfcfbd8b010ecb6ea1988f70594cbc10b400e7d9e8836e9f5c0', + ], + [ + 'Function', + 'sample.ts', + 'ast', + 'none', + 12, + '8c5780484a0f3cfcfbd8b010ecb6ea1988f70594cbc10b400e7d9e8836e9f5c0', + ], + [ + 'Function', + 'sample.ipynb', + 'ast', + 'full', + 12, + '04670833ee0a82f4e243bf065320bed8a6d2218168b7be617eabe6db144e8eb3', + ], + ] as const)( + 'matches AST-base %s %s %s %s inputs', + async (label, filePath, chunkStrategy, contentRetention, count, digest) => { + const content = + label === 'Class' + ? 'class Sample {\n field = 42;\n run() { return "BODY_EXCLUDED"; }\n}' + : filePath.endsWith('ipynb') + ? 'def Sample():\n' + + Array.from({ length: 45 }, (_, i) => ` value${i} = "雪😀"`).join('\n') + + '\n return value44' + : 'function Sample() {\n' + + Array.from({ length: 45 }, (_, i) => ` const value${i} = "雪😀";`).join('\n') + + '\n return value44;\n}'; + const node = { + id: `${label}:${filePath}:Sample`, + name: 'Sample', + label, + filePath, + content, + startLine: 10, + endLine: 10 + content.split('\n').length - 1, + }; + const config = resolveEmbeddingConfig({ + chunkSize: 100, + overlap: 20, + maxInputChars: 256, + chunkStrategy, + }); + const release = vi.fn(); + const readSource = vi.fn(async () => ({ content, release })); + const inputs = []; + for await (const input of iterateEmbeddingInputs(node, config, { + source: { repoPath: '/unused', fileHashes: {}, contentRetention }, + readSource, + })) + inputs.push(input); + expect(inputs).toHaveLength(count); + expect(createHash('sha256').update(JSON.stringify(inputs)).digest('hex')).toBe(digest); + expect(release).toHaveBeenCalledTimes(contentRetention === 'full' ? 1 : 0); + expect(node).not.toHaveProperty('classDeclaration'); + if (label === 'Class') + expect(inputs.map((input) => input.text).join('\n')).not.toContain('BODY_EXCLUDED'); + }, + ); + + it('releases expanded source when iteration is cancelled at a chunk boundary', async () => { + const node = { + ...fingerprintNode('Function', 'function sample() ... [truncated]'), + name: 'sample', + }; + const release = vi.fn(); + const iterator = iterateEmbeddingInputs(node, resolveEmbeddingConfig({}), { + source: { repoPath: '/unused', fileHashes: {}, contentRetention: 'full' }, + readSource: async () => ({ + content: 'function sample() {\n' + ' consume();\n'.repeat(1000) + '}', + release, + }), + }); + expect((await iterator.next()).done).toBe(false); + expect(release).not.toHaveBeenCalled(); + await iterator.return(undefined); + expect(release).toHaveBeenCalledOnce(); + }); it.each(['short', 'x'.repeat(2401)])( 'rejects a negative overlap without skipping source text', (content) => { @@ -29,18 +153,19 @@ describe('bounded embedding text preserves existing vectors', () => { 'export function 漢() {\r\n return "😀"; \r\n\r\n\r\n}\u3000\n', '69f18eaa2f90496499546d329389cfbbaf05b5ed', ], - ['File', 'a \t\r\n\n\n'.repeat(100000), '8750197e52b3b2d3ce89913a1946ff377f3b47e6'], + ['File', 'a \t\r\n\n\n'.repeat(100000), 'bcdad0b0fb7dda9b85406be8c775ef1df3ac060a'], [ 'Class', 'export class Example {\n value: string;\n method() { return "😀"; }\n}\n', - '9559301b3f875312190e654e10b16734631a1dee', + '05ce2f0a472c683379947bfe495db27a24e9c401', ], [ 'TypeAlias', 'type Huge = ' + '"漢😀" | '.repeat(40000) + 'never;\n', 'b0b393e258779fafede851f76d33d0831695e6bb', ], - ])('keeps the pre-change v7 %s fingerprint', (label, content, expected) => { + // Captured independently from AST base 6895f91726a3fc2f82add44c4bb0287a5c6b28d3. + ])('keeps the AST-base %s fingerprint', (label, content, expected) => { expect(contentHashForNode(fingerprintNode(label, content))).toBe(expected); }); From 7e881eddcbbbd37bb9906dc0de05228670621bb6 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 12:43:33 +0000 Subject: [PATCH 35/67] chore: run checks for the rebased stack From 51e2a211b7197de6838b6ad25681c00e3f2b2d31 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 13:20:28 +0000 Subject: [PATCH 36/67] refactor(embeddings): simplify reuse plumbing Name reuse-validation options and share the owned-directory loop. Preserve callback order, directory sync, and ownership checks. --- .../src/core/embeddings/embedding-inputs.ts | 7 +++++++ .../core/embeddings/embedding-job-runner.ts | 18 +++++------------ gitnexus/src/core/embeddings/embedding-job.ts | 20 +++++++------------ .../src/core/lbug/legacy-vector-source.ts | 11 +++++----- gitnexus/src/core/lbug/vector-store.ts | 7 +++---- gitnexus/src/storage/embedding-artifact.ts | 19 +++++++----------- gitnexus/src/storage/embedding-job.ts | 19 ++++++++++++++++++ .../integration/legacy-vector-export.test.ts | 2 +- .../integration/legacy-vector-source.test.ts | 12 +++++------ 9 files changed, 61 insertions(+), 54 deletions(-) diff --git a/gitnexus/src/core/embeddings/embedding-inputs.ts b/gitnexus/src/core/embeddings/embedding-inputs.ts index 4aafebac1..3e81667c9 100644 --- a/gitnexus/src/core/embeddings/embedding-inputs.ts +++ b/gitnexus/src/core/embeddings/embedding-inputs.ts @@ -22,6 +22,13 @@ export interface EmbeddingInput { endLine: number; } +export interface EmbeddingReuseOptions { + node: EmbeddableNode; + prepareInputValidation?: () => Promise; + inputs?: () => AsyncIterable; + matchesHash?: (stored: string) => Promise; +} + export async function* iterateEmbeddingInputs( indexedNode: EmbeddableNode, config: EmbeddingConfig, diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index aa4d1fff9..7fc48e629 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -15,7 +15,7 @@ import { type EmbeddingWorkUnit, } from './embedding-work-manifest.js'; import { contentHashForNode, embedInputRanges, isInputRejection } from './embedding-pipeline.js'; -import { iterateEmbeddingInputs, type EmbeddingInput } from './embedding-inputs.js'; +import { iterateEmbeddingInputs, type EmbeddingReuseOptions } from './embedding-inputs.js'; import { EmbeddingSourceReader, needsEmbeddingSource, @@ -31,10 +31,7 @@ export interface EmbeddingReuseSource { reusable( nodeId: string, contentHash: string, - node: EmbeddableNode, - prepareInputValidation?: () => Promise, - inputs?: () => AsyncIterable, - matchesHash?: (stored: string) => Promise, + options: EmbeddingReuseOptions, ): Promise; readChunks: VectorStore['readChunks']; } @@ -349,14 +346,12 @@ export async function runEmbeddingJob( continue; } } - const reusable = await options.reuse?.reusable( - node.id, - input.contentHash, + const reusable = await options.reuse?.reusable(node.id, input.contentHash, { node, prepareInputValidation, - () => nodeInputs(node), + inputs: () => nodeInputs(node), matchesHash, - ); + }); if (reusable) reusableNodes.set(node.id, reusable); else input.contentHash = baseHash; } @@ -378,9 +373,6 @@ export async function runEmbeddingJob( if (abort) break; const copyRowBytes = 32 * dimensions + 1024 + 4 * node.id.length; for (let start = 0; start < reusable.chunks; ) { - // Share one bounded transaction across neighboring reused nodes. - // Retain its reservation until the write finishes, including while - // validating the next node's historical input. if ( copyBatch.length && (copyBatch.length === 256 || diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index e44846826..55fc210fc 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -3,7 +3,11 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { createHash, randomBytes } from 'node:crypto'; import type { RepoMeta } from '../../storage/repo-meta.js'; -import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; +import { + createEmbeddingJobDirectories, + embeddingJobDirectory, + syncEmbeddingJobDirectory, +} from '../../storage/embedding-job.js'; import { adoptEmbeddingArtifact, embeddingArtifactSpaceKey, @@ -87,18 +91,8 @@ async function existsOwnedFile(file: string): Promise { } async function jobDirectory(root: string, identity: string): Promise { - let directory = await embeddingJobDirectory(root); - for (const component of ['embeddings', identity]) { - const next = path.join(directory, component); - try { - await fs.mkdir(next, { mode: 0o700 }); - await syncEmbeddingJobDirectory(directory); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - } - directory = await embeddingJobDirectory(next); - } - return directory; + const directory = await embeddingJobDirectory(root); + return createEmbeddingJobDirectories(directory, ['embeddings', identity]); } /** A preflight interrupted before any native/job artifact owns no paid work. diff --git a/gitnexus/src/core/lbug/legacy-vector-source.ts b/gitnexus/src/core/lbug/legacy-vector-source.ts index e9688d637..eccb59f46 100644 --- a/gitnexus/src/core/lbug/legacy-vector-source.ts +++ b/gitnexus/src/core/lbug/legacy-vector-source.ts @@ -15,7 +15,10 @@ import { import { assertValidGraphNodeId } from '../graph/node-identity.js'; import type { EmbeddingIdentity } from '../embeddings/embedding-identity.js'; import { resolveEmbeddingConfig } from '../embeddings/config.js'; -import { iterateEmbeddingInputs, type EmbeddingInput } from '../embeddings/embedding-inputs.js'; +import { + iterateEmbeddingInputs, + type EmbeddingReuseOptions, +} from '../embeddings/embedding-inputs.js'; import { isShortLabel, type EmbeddableNode, type EmbeddingConfig } from '../embeddings/types.js'; export interface LegacyVectorSourceOptions { @@ -257,11 +260,9 @@ export class LegacyVectorSource { async reusable( nodeId: string, contentHash: string, - node: EmbeddableNode, - prepareInputValidation?: () => Promise, - inputs?: () => AsyncIterable, - matchesHash?: (stored: string) => Promise, + options: EmbeddingReuseOptions, ): Promise { + const { node, prepareInputValidation, inputs, matchesHash } = options; this.acquire(); const originalMethods = node.methodNames, originalFields = node.fieldNames; diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 0b04dbcf0..319d7e574 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -11,6 +11,7 @@ import { ExtensionManager } from './extension-loader.js'; import { assertValidGraphNodeId } from '../graph/node-identity.js'; import { embeddingJobDirectory } from '../../storage/embedding-job.js'; import { getExactScanLimit } from '../platform/capabilities.js'; +import type { EmbeddingReuseOptions } from '../embeddings/embedding-inputs.js'; export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v2'; const BLOCK_ROWS = 256; @@ -631,11 +632,9 @@ export class VectorStore { async reusable( nodeId: string, contentHash: string, - _node?: unknown, - _prepare?: unknown, - _inputs?: unknown, - matchesHash?: (stored: string) => Promise, + options?: Pick, ): Promise { + const matchesHash = options?.matchesHash; return this.exclusive(async () => { const receipt = await this.receipt(nodeId); if ( diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts index 4a69feee8..e997fd02b 100644 --- a/gitnexus/src/storage/embedding-artifact.ts +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -4,6 +4,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { createHash } from 'node:crypto'; import { + createEmbeddingJobDirectories, embeddingJobDirectory, readEmbeddingJobRecord, writeEmbeddingJobRecord, @@ -122,18 +123,12 @@ function record(value: unknown, space: EmbeddingArtifactSpace): EmbeddingArtifac } async function spaceDirectory(root: string, space: EmbeddingArtifactSpace): Promise { - let directory = await embeddingJobDirectory(root); - for (const component of ['embeddings', 'spaces', embeddingArtifactSpaceKey(space)]) { - const next = path.join(directory, component); - try { - await fs.mkdir(next, { mode: 0o700 }); - await syncEmbeddingJobDirectory(directory); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - } - directory = await embeddingJobDirectory(next); - } - return directory; + const directory = await embeddingJobDirectory(root); + return createEmbeddingJobDirectories(directory, [ + 'embeddings', + 'spaces', + embeddingArtifactSpaceKey(space), + ]); } async function optionalRecord(directory: string, name: string): Promise { diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index eae965ccb..4efb57bfe 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -56,6 +56,25 @@ export async function embeddingJobDirectory(directory: string): Promise return fs.realpath(directory); } +/** Root is already canonical and validated; validate each child before traversing it. */ +export async function createEmbeddingJobDirectories( + root: string, + components: readonly string[], +): Promise { + let directory = root; + for (const component of components) { + const next = path.join(directory, component); + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + /** Flush directory entries where the platform supports opening directories. */ export async function syncEmbeddingJobDirectory(directory: string): Promise { if (process.platform === 'win32') return; diff --git a/gitnexus/test/integration/legacy-vector-export.test.ts b/gitnexus/test/integration/legacy-vector-export.test.ts index 4d52999c4..cf216b2f5 100644 --- a/gitnexus/test/integration/legacy-vector-export.test.ts +++ b/gitnexus/test/integration/legacy-vector-export.test.ts @@ -131,7 +131,7 @@ it('exports only the vector table and retains a compatible complete group withou store, receipt.excludedIds, ); - const group = await source.reusable(node.id, f.hash, node); + const group = await source.reusable(node.id, f.hash, { node }); expect(group?.chunks).toBe(1); expect(await source.readChunks(group!, 0, 1)).toEqual([ { chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0, 0, 0] }, diff --git a/gitnexus/test/integration/legacy-vector-source.test.ts b/gitnexus/test/integration/legacy-vector-source.test.ts index 38ed546cf..6ecfdf071 100644 --- a/gitnexus/test/integration/legacy-vector-source.test.ts +++ b/gitnexus/test/integration/legacy-vector-source.test.ts @@ -160,7 +160,7 @@ describe('verified legacy vector reuse', () => { expect(Math.max(...f.chunks.map((chunk) => chunk.endLine))).toBeLessThan(original.endLine); const reuse = await f.open(); const moved = { ...original, startLine: 11, endLine: 210 }; - expect(await reuse.reusable(moved.id, f.hash, moved)).toMatchObject({ + expect(await reuse.reusable(moved.id, f.hash, { node: moved })).toMatchObject({ complete: true, sourceStartLine: 1, sourceEndLine: undefined, @@ -236,7 +236,7 @@ describe('verified legacy vector reuse', () => { expect(new Set(f.chunks.map((c) => `${c.startLine}:${c.endLine}`)).size).toBe(1); const log = vi.fn(); const reuse = await f.open({ recordedInputIdentity: undefined, log }); - expect(await reuse.reusable(input.id, f.hash, input)).toBeNull(); + expect(await reuse.reusable(input.id, f.hash, { node: input })).toBeNull(); expect(log).toHaveBeenCalledOnce(); expect(log.mock.calls[0][0]).toContain('chunk size/overlap'); }); @@ -244,7 +244,7 @@ describe('verified legacy vector reuse', () => { const input = node('Const'); const f = await fixture(input); const reuse = await f.open({ recordedInputIdentity: undefined }); - expect(await reuse.reusable(input.id, f.hash, input)).toMatchObject({ + expect(await reuse.reusable(input.id, f.hash, { node: input })).toMatchObject({ chunks: f.chunks.length, }); expect(reuse.options.recordedInputIdentity).toBeUndefined(); @@ -252,13 +252,13 @@ describe('verified legacy vector reuse', () => { it('rejects changed hashes, holes, and groups in an unsafe checkpoint window', async () => { const input = node('File', 'x'.repeat(1000)); const f = await fixture(input, (rows) => rows.filter((row) => row.chunkIndex !== 1)); - expect(await (await f.open()).reusable(input.id, f.hash, input)).toBeNull(); + expect(await (await f.open()).reusable(input.id, f.hash, { node: input })).toBeNull(); const intact = await fixture(input); const excluded = await intact.open({}, [input.id]); - expect(await excluded.reusable(input.id, intact.hash, input)).toBeNull(); + expect(await excluded.reusable(input.id, intact.hash, { node: input })).toBeNull(); const changed = await fixture(node('Const')); expect( - await (await changed.open()).reusable('Const:a.txt:n', 'different', node('Const')), + await (await changed.open()).reusable('Const:a.txt:n', 'different', { node: node('Const') }), ).toBeNull(); }); }); From fe685375fadfd98e0a851d70c932327654b88081 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 16:12:44 +0000 Subject: [PATCH 37/67] test: fix recovery and Windows CI fixtures Distinguish legacy checkpoints without input provenance from verified source-aware chunks. Check the embed API at its shared-phase boundary and model pathname races without rename-over-open on Windows. --- .../analyze-staged-embedding-recovery.test.ts | 50 +++++++++++++++++++ gitnexus/test/unit/api-fts-mode.test.ts | 20 +++++--- .../unit/embedding-job-file-security.test.ts | 10 +++- .../unit/shared-store-read-security.test.ts | 19 +++++-- 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts index 3ff345d09..7364e4b66 100644 --- a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts +++ b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts @@ -48,6 +48,7 @@ interface Run { let root: string; let stoppedRepo: string; +let sourceAwareRepo: string; let server: http.Server; let endpoint: string; let submitted: string[] = []; @@ -270,6 +271,40 @@ function expectLegacyReuse(texts: string[], unfinishedInputs: number): void { expect(lastOutput).toContain('Legacy multi-chunk input provenance is unknown'); } +function removeLegacyInputProvenance(repo: string): void { + const stagingFile = readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile; + if (!stagingFile) throw new Error('fixture has no retained generation'); + // The retained core path now writes source-aware hashes. Downgrade only the + // long group's input digest to exercise checkpoints predating that receipt. + const script = ` + const native = await import(${JSON.stringify(nativeUrl)}); + const { default: lbug } = await import('@ladybugdb/core'); + const handle = await native.openLbugConnection(lbug, ${JSON.stringify(path.join(repo, '.gitnexus', stagingFile))}, { + bufferPoolBytes: 64 * 1024 ** 2, threads: 1, + }); + try { + const nodeId = 'Function:functions.ts:longRecoverable'; + const query = await handle.conn.prepare('MATCH (e:CodeEmbedding {nodeId: $nodeId}) RETURN e.contentHash AS contentHash'); + const result = await handle.conn.execute(query, { nodeId }); + const rows = await result.getAll(); + result.close(); + const hash = rows[0]?.contentHash; + if (rows.length < 2 || !/^[a-f0-9]{40}:[a-f0-9]{40}$/.test(hash) || rows.some(row => row.contentHash !== hash)) + throw new Error('Expected a complete source-aware multi-chunk fixture'); + const update = await handle.conn.prepare('MATCH (e:CodeEmbedding {nodeId: $nodeId}) SET e.contentHash = $contentHash'); + (await handle.conn.execute(update, { nodeId, contentHash: hash.slice(0, 40) })).close(); + console.log('LEGACY_INPUT_PROVENANCE_REMOVED'); + } finally { await native.closeLbugConnection(handle); } + `; + const result = spawnSync( + process.execPath, + ['--import', tsxLoaderUrl(), '--input-type=module', '-e', script], + { cwd: packageRoot, env: cliEnv(repo), encoding: 'utf8', timeout: 30_000 }, + ); + expect(result.status, `${result.error ?? ''}\n${result.stderr}\n${result.stdout}`).toBe(0); + expect(result.stdout).toContain('LEGACY_INPUT_PROVENANCE_REMOVED'); +} + beforeAll(async () => { if (process.platform === 'win32') return; root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-staged-recovery-e2e-')); @@ -364,6 +399,8 @@ beforeAll(async () => { expect(completed.filter((text) => text.includes('longRecoverable')).length).toBeGreaterThan(1); expect(activeWindowCompleted).toHaveLength(64); expect(held.length).toBeGreaterThan(0); + sourceAwareRepo = cloneStoppedRepo('source-aware'); + removeLegacyInputProvenance(stoppedRepo); gateResolve = undefined; stopAfterCheckpoint = false; }, DEADLINE * 2); @@ -387,6 +424,19 @@ describe.skipIf(process.platform === 'win32')( 'interrupted staged embedding recovery (real CLI and native DB)', { concurrent: false }, () => { + it( + 'reuses verified source-aware multi-chunk groups without repeat provider work', + async () => { + const texts = await successfulAnalyze(sourceAwareRepo); + expect(texts.filter((text) => durableTexts.includes(text))).toEqual([]); + expect(texts).toHaveLength(128); + for (const text of held) expect(texts).toContain(text); + for (const text of activeWindowCompleted) expect(texts).toContain(text); + expectPublishedComplete(sourceAwareRepo, NODE_COUNT); + }, + DEADLINE, + ); + it.each([ ['plain', []], ['forced', ['--force', '--embeddings']], diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 23f4bcbd0..bd58aa27a 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -211,7 +211,7 @@ it.each(['analyzing', 'failed'])( ); describe('POST /api/embed staged recovery preflight', () => { - it('passes indexed source identity from the checkout and reports source failures', async () => { + it('hands the indexed storage slot to the shared phase and reports source failures', async () => { const fileHashes = { 'sample.ts': 'a'.repeat(64) }; mocks.loadMeta.mockResolvedValue({ repoPath: entry.path, @@ -219,14 +219,20 @@ describe('POST /api/embed staged recovery preflight', () => { contentRetention: 'full', stats: { embeddings: 3 }, }); - mocks.runEmbeddingPipeline.mockRejectedValue(new Error('source fingerprint mismatch')); + mocks.runAnalysisEmbeddingPhase.mockRejectedValueOnce(new Error('source fingerprint mismatch')); await invoke('/api/embed'); await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); - expect(mocks.runEmbeddingPipeline.mock.calls[0]?.[6].source).toEqual({ - repoPath: entry.path, - fileHashes, - contentRetention: 'full', - }); + // The shared phase reads source identity from this slot's published metadata. + // embedding-job-controller.test.ts exercises that read with the real controller. + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + metadataDirectory: entry.storagePath, + graphPath: path.join(entry.storagePath, 'lbug'), + nodeLimit: 0, + signal: expect.any(AbortSignal), + }), + ); + expect(mocks.runEmbeddingPipeline).not.toHaveBeenCalled(); expect(entry.path).not.toBe(entry.storagePath); expect(mocks.updateJob).toHaveBeenCalledWith( 'embed-job', diff --git a/gitnexus/test/unit/embedding-job-file-security.test.ts b/gitnexus/test/unit/embedding-job-file-security.test.ts index e302cb4a7..dab003058 100644 --- a/gitnexus/test/unit/embedding-job-file-security.test.ts +++ b/gitnexus/test/unit/embedding-job-file-security.test.ts @@ -60,17 +60,25 @@ it('rejects a different regular file at the opened path when O_NOFOLLOW is unava const { readEmbeddingJobRecord: readWithoutNoFollow } = await import('../../src/storage/embedding-job.js'); const open = fs.open.bind(fs); + const lstat = fs.lstat.bind(fs); let close: ReturnType | undefined; + let read: ReturnType | undefined; vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { const handle = await open(target, flags, mode); if (target === file) { close = vi.spyOn(handle, 'close'); - await fs.rename(path.join(replacement, 'job.json'), file); + read = vi.spyOn(handle, 'read'); + // Model a changed directory entry with real inode stats. Windows cannot + // rename over an open file, so a rename would fail before testing the guard. + vi.spyOn(fs, 'lstat').mockImplementation((target, options) => + lstat(target === file ? path.join(replacement, 'job.json') : target, options), + ); } return handle; }); await expect(readWithoutNoFollow(directory, 'job.json')).rejects.toThrow('changed during open'); expect(close).toHaveBeenCalledOnce(); + expect(read).not.toHaveBeenCalled(); }); it.skipIf(process.platform === 'win32')( diff --git a/gitnexus/test/unit/shared-store-read-security.test.ts b/gitnexus/test/unit/shared-store-read-security.test.ts index 88ea59614..dd235a418 100644 --- a/gitnexus/test/unit/shared-store-read-security.test.ts +++ b/gitnexus/test/unit/shared-store-read-security.test.ts @@ -45,16 +45,27 @@ it('reads the opened metadata snapshot and invalidates it after pathname replace const { slot, metadata } = fixture(); const replacement = path.join(slot, 'replacement.json'); fs.writeFileSync(replacement, JSON.stringify({ graphPath: second })); + // Model a pathname swap while keeping both real files available: Windows + // refuses rename-over-open, but descriptor reads must remain pinned there too. + let replaced = false; + const open = fs.openSync.bind(fs); + const stat = fs.lstatSync.bind(fs); + vi.spyOn(fs, 'openSync').mockImplementation((file, flags, mode) => + open(replaced && file === metadata ? replacement : file, flags, mode), + ); + vi.spyOn(fs, 'lstatSync').mockImplementation((file, options) => + stat(replaced && file === metadata ? replacement : file, options), + ); const read = fs.readFileSync.bind(fs); const spy = vi.spyOn(fs, 'readFileSync').mockImplementation((...args) => { - if (typeof args[0] === 'number') { - spy.mockRestore(); - fs.renameSync(replacement, metadata); - } + replaced = true; + if (args[0] === metadata) args[0] = replacement; return read(...args); }); expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); expect(resolveGraphPath(slot)).toBe(path.join(slot, second)); + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.calls.every(([file]) => typeof file === 'number')).toBe(true); }); it('preserves a relative storage path while validating its generation', () => { From 701fe8b7be15d05bd4e6661c7bc955d192d1ea53 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 18:37:38 +0000 Subject: [PATCH 38/67] fix: close resource admission and CI gaps Keep live sidecars charged after a reap timeout. Let sync and HTTP jobs use automatic host budgets instead of overcommitting to default V8 limits. Explain the incompatible native zero-pool override, clean up failed HTTP fixtures, and run the native vector suites across platforms. --- RUNBOOK.md | 7 ++- gitnexus/scripts/cross-platform-tests.ts | 3 + gitnexus/src/cli/embeddings-sync.ts | 2 - gitnexus/src/core/analysis-host-resources.ts | 3 + gitnexus/src/core/analysis-resource-budget.ts | 5 ++ .../embeddings/embedding-sidecar-client.ts | 2 +- gitnexus/src/server/api.ts | 2 - .../analyze-http-embedding-phases.test.ts | 58 ++++++++++--------- .../test/unit/analysis-host-resources.test.ts | 15 ++++- gitnexus/test/unit/api-fts-mode.test.ts | 12 ++++ .../unit/embedding-sidecar-client.test.ts | 3 +- .../test/unit/embeddings-sync-command.test.ts | 21 +++++++ gitnexus/vitest.config.ts | 3 + 13 files changed, 100 insertions(+), 36 deletions(-) diff --git a/RUNBOOK.md b/RUNBOOK.md index 0b9d18e47..512406278 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -310,8 +310,11 @@ GITNEXUS_LBUG_BUFFER_POOL_SIZE=3221225472 npx gitnexus analyze --force The override bypasses automatic pool sizing. Budget for the Node heap, parse workers, other native allocations, and the OS alongside it; a larger pool can -exhaust the host. `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` restores LadybugDB's native -80%-of-RAM default, not an unlimited supply of memory. If memory is insufficient, +exhaust the host. Outside bounded analysis, `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` +selects LadybugDB's native 80%-of-RAM default. Analysis and embedding admission +reject zero because that native allocation cannot fit alongside their other +reservations. Unset the variable for automatic sizing, or set an explicit byte +limit of at least 8 MiB that fits the resource budget. If memory is insufficient, exclude generated/vendor directories or use a larger host. A measured graph-write collapse aborts a staged build before publication, diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index f235d3400..627e86bc7 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -167,6 +167,9 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/embedding-vector-store.test.ts', + 'test/integration/legacy-vector-reuse-index.test.ts', + 'test/integration/legacy-vector-export.test.ts', 'test/integration/java-overload-impact-persistence.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/impact-private-type-file-seed.test.ts', diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts index 02369b8da..bbf544057 100644 --- a/gitnexus/src/cli/embeddings-sync.ts +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -4,7 +4,6 @@ import { resolveGraphPath } from '../storage/shared-store.js'; import { resolveHostAnalysisResources } from '../core/analysis-host-resources.js'; import { readPreparedAnalysis } from '../core/analysis-publication.js'; import path from 'node:path'; -import { getHeapStatistics } from 'node:v8'; import { cliInfo } from './cli-message.js'; import { getGitRoot } from '../storage/git.js'; import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js'; @@ -36,7 +35,6 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = const resources = resolveHostAnalysisResources({ includeParsing: false, includeEmbeddings: true, - mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), }); const lock = await acquireAnalysisSlotLock(metaDir, { sweep: false }); try { diff --git a/gitnexus/src/core/analysis-host-resources.ts b/gitnexus/src/core/analysis-host-resources.ts index d5f11c088..18cf6e924 100644 --- a/gitnexus/src/core/analysis-host-resources.ts +++ b/gitnexus/src/core/analysis-host-resources.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { effectiveRamBytes } from './ingestion/utils/effective-ram.js'; import { resolveAnalysisResourceBudget, + UNBOUNDED_GRAPH_POOL_MESSAGE, type AnalysisResourceBudget, type AnalysisResourceRequest, } from './analysis-resource-budget.js'; @@ -14,6 +15,8 @@ function positiveEnvironment(name: string): number | undefined { const raw = process.env[name]; if (raw === undefined || raw.trim() === '') return undefined; const value = Number(raw); + if (name === 'GITNEXUS_LBUG_BUFFER_POOL_SIZE' && value === 0) + throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE); if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive safe integer for bounded analysis`); return value; diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 021c774f6..c5eee9f93 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -69,10 +69,15 @@ function bytes(mb: number, label: string): number { return integer(integer(mb, label, 0) * MIB, `${label} in bytes`, 0); } +export const UNBOUNDED_GRAPH_POOL_MESSAGE = + "Zero graph pool bytes selects LadybugDB's native 80%-of-RAM default, which bounded analysis cannot reserve. " + + 'Unset GITNEXUS_LBUG_BUFFER_POOL_SIZE for automatic sizing or set a positive byte limit of at least 8388608.'; + /** Resolve before opening a writable database or starting parser/model work. */ export function resolveAnalysisResourceBudget( request: AnalysisResourceRequest, ): AnalysisResourceBudget { + if (request.graphPoolBytes === 0) throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE); const memory = integer(request.effectiveMemoryBytes, 'effectiveMemoryBytes'); const cpus = integer(request.availableCpus, 'availableCpus'); const embeddingThreads = diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts index 690c03559..457fede65 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts @@ -262,7 +262,7 @@ const request = ( }; const spawnSidecar = (heapLimitMb?: number): ChildProcess => { - if (runtimeReservation && liveChildren.size > 0) { + if (liveChildren.size > 0) { throw new Error('Previous embedding sidecar has not released its native resource reservation'); } const proc = forkImpl(sidecarScriptPath(), [], { diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index f2b47150f..8891369b2 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -2192,11 +2192,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => sweepStagingArtifacts(storagePath); const { resolveHostAnalysisResources } = await import('../core/analysis-host-resources.js'); - const { getHeapStatistics } = await import('node:v8'); const resources = resolveHostAnalysisResources({ includeParsing: false, includeEmbeddings: true, - mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), }); if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) throw new Error('The shared graph this repository reads is gone. Re-run analyze.'); diff --git a/gitnexus/test/integration/analyze-http-embedding-phases.test.ts b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts index 2207a84ac..2c0c697cb 100644 --- a/gitnexus/test/integration/analyze-http-embedding-phases.test.ts +++ b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts @@ -74,30 +74,30 @@ it('serves the published graph during an HTTP outage and resumes through /api/em }; child.stdout?.on('data', collect); child.stderr?.on('data', collect); - const exited = new Promise((resolve) => child.once('exit', resolve)); - const port = await new Promise((resolve, reject) => { - child.once('message', (message) => resolve((message as { port: number }).port)); - child.once('error', reject); - void exited.then((code) => reject(new Error(`Server exited ${code}: ${output}`))); - }); - const request = async (route: string, body?: unknown) => { - const response = await fetch(`http://127.0.0.1:${port}${route}`, { - method: body === undefined ? 'GET' : 'POST', - headers: { 'content-type': 'application/json' }, - ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }); - const value = (await response.json()) as Record; - expect(response.ok, JSON.stringify(value) + '\n' + output).toBe(true); - return value; - }; - const until = async (check: () => Promise) => { - const end = Date.now() + 90_000; - while (!(await check())) { - if (Date.now() > end) throw new Error(`Timed out: ${output}`); - await new Promise((resolve) => setTimeout(resolve, 100)); - } - }; + const exited = new Promise((resolve) => child.once('close', resolve)); try { + const port = await new Promise((resolve, reject) => { + child.once('message', (message) => resolve((message as { port: number }).port)); + child.once('error', reject); + void exited.then((code) => reject(new Error(`Server exited ${code}: ${output}`))); + }); + const request = async (route: string, body?: unknown) => { + const response = await fetch(`http://127.0.0.1:${port}${route}`, { + method: body === undefined ? 'GET' : 'POST', + headers: { 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const value = (await response.json()) as Record; + expect(response.ok, JSON.stringify(value) + '\n' + output).toBe(true); + return value; + }; + const until = async (check: () => Promise) => { + const end = Date.now() + 90_000; + while (!(await check())) { + if (Date.now() > end) throw new Error(`Timed out: ${output}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }; const started = await request('/api/analyze', { path: repo, embeddings: true }); const poll = () => request(`/api/analyze/${started.jobId}`); await until(async () => (await poll()).status === 'analyzing'); @@ -145,9 +145,13 @@ it('serves the published graph during an HTTP outage and resumes through /api/em expect((await loadMeta(slot))?.graphGeneration).toEqual(generation); } finally { releaseOutage(); - if (child.connected) child.send('shutdown'); - expect(await exited, output).toBe(0); - await new Promise((resolve) => provider.close(() => resolve())); - await fs.rm(root, { recursive: true, force: true }); + try { + if (child.connected) child.send('shutdown'); + expect(await exited, output).toBe(0); + } finally { + provider.closeAllConnections(); + await new Promise((resolve) => provider.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + } } }, 180_000); diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts index 8a0e9f88e..f8ce5e2c3 100644 --- a/gitnexus/test/unit/analysis-host-resources.test.ts +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -63,7 +63,20 @@ it('honors bounded worker, pool, and embedding-thread overrides', () => { expect(budget.embedding.nativeThreads).toBe(8); }); -it.each(['0', '-1', '1.5', 'NaN'])( +it.each(['0', ' 0 ', '0.0'])( + 'explains why the native default pool %s cannot be reserved', + (value) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', value); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'bounded analysis cannot reserve', + ); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'Unset GITNEXUS_LBUG_BUFFER_POOL_SIZE', + ); + }, +); + +it.each(['-1', '1.5', 'NaN'])( 'rejects an unbounded or malformed native pool override %s', (value) => { vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', value); diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index bd58aa27a..6fa9e8d77 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -5,6 +5,17 @@ import os from 'node:os'; import path from 'node:path'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ + effectiveRamBytes: () => 8 * 1024 ** 3, +})); +vi.mock('node:v8', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getHeapStatistics: () => ({ ...actual.getHeapStatistics(), heap_size_limit: 64 * 1024 ** 3 }), + }; +}); + const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), saveMeta: vi.fn(), @@ -229,6 +240,7 @@ describe('POST /api/embed staged recovery preflight', () => { metadataDirectory: entry.storagePath, graphPath: path.join(entry.storagePath, 'lbug'), nodeLimit: 0, + resources: expect.objectContaining({ totalBytes: 6 * 1024 ** 3 }), signal: expect.any(AbortSignal), }), ); diff --git a/gitnexus/test/unit/embedding-sidecar-client.test.ts b/gitnexus/test/unit/embedding-sidecar-client.test.ts index 3a11cd75d..d417c74fb 100644 --- a/gitnexus/test/unit/embedding-sidecar-client.test.ts +++ b/gitnexus/test/unit/embedding-sidecar-client.test.ts @@ -227,7 +227,7 @@ describe('embedding sidecar client', () => { it('does not certify a timed-out reap or allocate another job over the retained process', async () => { vi.useFakeTimers(); - const { reserveEmbeddingSidecar } = + const { reserveEmbeddingSidecar, ensureEmbeddingSidecar } = await import('../../src/core/embeddings/embedding-sidecar-client.js'); const config = resolveEmbeddingConfig({ threads: 1 }); const first = await reserveEmbeddingSidecar(config, 512); @@ -239,6 +239,7 @@ describe('embedding sidecar client', () => { ); await vi.advanceTimersByTimeAsync(5_001); await rejected; + await expect(ensureEmbeddingSidecar()).rejects.toThrow('native resource reservation'); await expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow( 'native resource reservation', ); diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 6da1df66f..14e447485 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -101,6 +101,17 @@ vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ const IDENTITY = { model: 'test-model', dimensions: 768, provider: 'local' } as const; +vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ + effectiveRamBytes: () => 8 * 1024 ** 3, +})); +vi.mock('node:v8', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getHeapStatistics: () => ({ ...actual.getHeapStatistics(), heap_size_limit: 64 * 1024 ** 3 }), + }; +}); + const BASE_META = { repoPath: '/tmp/emb-sync-repo', lastCommit: 'abc123', @@ -200,6 +211,16 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); + it('keeps automatic reservations within host memory despite a larger default V8 limit', async () => { + await store(); + await run(); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + resources: expect.objectContaining({ totalBytes: 6 * 1024 ** 3 }), + }), + ); + }); + it('acquires the index lock before re-reading metadata and releases it in finally', async () => { const { dir } = await store(); const order: string[] = []; diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 1db21a4fd..08085f6f9 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -4,6 +4,7 @@ import PerfSequencer from './test/helpers/perf-sequencer.js'; // Keep native tests in exactly one project (TESTING.md). const EMBEDDING_VECTOR_STORE_TEST = 'test/integration/embedding-vector-store.test.ts'; const LEGACY_VECTOR_REUSE_TEST = 'test/integration/legacy-vector-reuse-index.test.ts'; +const LEGACY_VECTOR_EXPORT_TEST = 'test/integration/legacy-vector-export.test.ts'; export default defineConfig({ test: { @@ -84,6 +85,7 @@ export default defineConfig({ include: [ EMBEDDING_VECTOR_STORE_TEST, LEGACY_VECTOR_REUSE_TEST, + LEGACY_VECTOR_EXPORT_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', @@ -204,6 +206,7 @@ export default defineConfig({ exclude: [ EMBEDDING_VECTOR_STORE_TEST, LEGACY_VECTOR_REUSE_TEST, + LEGACY_VECTOR_EXPORT_TEST, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', From 8c1b489c962a223b77567c21dcd1a9f2c9788851 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 19:13:48 +0000 Subject: [PATCH 39/67] fix: preserve analysis resource ownership Pin admitted model settings and retain worker and supervisor reservations. Recheck canonical directory identity before receipt reads or writes. Canonicalize the native publication fixture for macOS and Windows. --- gitnexus/src/cli/analyze-watch.ts | 11 +++- .../embeddings/embedding-sidecar-client.ts | 12 +++- .../src/core/ingestion/workers/worker-pool.ts | 6 +- gitnexus/src/storage/embedding-job.ts | 11 +++- .../embedding-vector-store.test.ts | 2 +- .../unit/embedding-job-file-security.test.ts | 20 ++++++ .../unit/embedding-sidecar-client.test.ts | 24 +++++++ gitnexus/test/unit/watch-cancellation.test.ts | 1 + .../unit/watch-supervisor-reservation.test.ts | 62 +++++++++++++++++++ .../unit/worker-pool-timeout-retire.test.ts | 29 +++++++++ 10 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 gitnexus/test/unit/watch-supervisor-reservation.test.ts diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index bb1e75c3c..f4382203d 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -27,7 +27,11 @@ import type { AnalyzeOptions } from './analyze-options.js'; import { ensureHeap } from './analyze.js'; import { cliError, cliInfo, cliWarn } from './cli-message.js'; import { parseIntegerOption } from './int-option.js'; -import { analysisMayEmbed, resolveHostAnalysisResources } from '../core/analysis-host-resources.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; import { formatInvalidProcessDetectionOverride, parseProcessDetectionBudgetStrings, @@ -195,6 +199,7 @@ export async function resolveWatchOptions( mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), workers: workerPoolSize, includeEmbeddings: await analysisMayEmbed(repoPath, { branch }), + supervisorBytes: Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES ?? 0), }); setEnvironment( @@ -495,6 +500,10 @@ export async function watchCommandWithRunnerIdentity( mainHeapMb: positiveInteger(cliOptions.memoryBudget, '--memory-budget'), workers: positiveInteger(cliOptions.workers, '--workers'), includeEmbeddings: await analysisMayEmbed(repoPath, { branch: cliOptions.branch }), + supervisorBytes: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES), }); if (cancelled) return; if ( diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts index 457fede65..a78e307db 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts @@ -108,7 +108,17 @@ const childEnv = (configPinned = false): NodeJS.ProcessEnv => { if (configPinned) { // The admitted, resolved config travels over IPC. An environment mutation // between admission and lazy model init must not override its allocations. - for (const name of ['THREADS', 'DEVICE', 'BATCH_SIZE', 'SUB_BATCH_SIZE', 'MAX_INPUT_CHARS']) { + for (const name of [ + 'THREADS', + 'DEVICE', + 'BATCH_SIZE', + 'SUB_BATCH_SIZE', + 'MAX_INPUT_CHARS', + 'MODEL_ID', + 'CHUNK_SIZE', + 'CHUNK_OVERLAP', + 'CHUNK_STRATEGY', + ]) { delete env[`GITNEXUS_EMBEDDING_${name}`]; } } diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index aebd4a0da..360da913b 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -1510,7 +1510,7 @@ export const createWorkerPool = ( }); // A message/messageerror proves the worker is executing JS again; an - // exit/error means the thread is gone. Either way `worker.terminate()` + // exit/error means native execution has stopped. Either way `worker.terminate()` // can no longer land mid-N-API call (#2432), so shutdown's drain may // stop waiting. function markSafeToTerminate() { @@ -1553,7 +1553,9 @@ export const createWorkerPool = ( const onRetiredError = () => { markSafeToTerminate(); - cleanupRetired(); + // The error event precedes physical exit. Keep the reservation and + // exit listener if termination subsequently rejects or stalls. + if (!requireDrain) cleanupRetired(); }; const onRetiredExit = () => { markSafeToTerminate(); diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 4efb57bfe..846662630 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -53,7 +53,16 @@ export async function embeddingJobDirectory(directory: string): Promise const stat = await fs.lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('Embedding job directory must be an owned directory'); - return fs.realpath(directory); + const canonical = await fs.realpath(directory); + const resolved = await fs.lstat(canonical); + if ( + !resolved.isDirectory() || + resolved.isSymbolicLink() || + resolved.dev !== stat.dev || + resolved.ino !== stat.ino + ) + throw new Error('Embedding job directory changed during resolution'); + return canonical; } /** Root is already canonical and validated; validate each child before traversing it. */ diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index ae23457ba..35555b9ac 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -1030,7 +1030,7 @@ describe('published vector readers', () => { temporary = await createTempDir(); vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(64 * 1024 * 1024)); - const root = temporary.dbPath; + const root = await fs.realpath(temporary.dbPath); const stage = path.join(root, `lbug.staging.${randomUUID()}`); const graph = await openLbugConnection(lbug, stage, { bufferPoolBytes: 64 * 1024 * 1024, diff --git a/gitnexus/test/unit/embedding-job-file-security.test.ts b/gitnexus/test/unit/embedding-job-file-security.test.ts index dab003058..335cf5882 100644 --- a/gitnexus/test/unit/embedding-job-file-security.test.ts +++ b/gitnexus/test/unit/embedding-job-file-security.test.ts @@ -28,6 +28,26 @@ afterEach(async () => { ); }); +it.each(['read', 'write'] as const)( + 'rejects a directory redirected during realpath before receipt %s', + async (operation) => { + const directory = await fixture({ owned: true }); + const outside = await fixture({ outside: true }); + const realpath = fs.realpath.bind(fs); + vi.spyOn(fs, 'realpath').mockImplementation((target, options) => + realpath(target === directory ? outside : target, options), + ); + const open = vi.spyOn(fs, 'open'); + await expect( + operation === 'read' + ? readEmbeddingJobRecord(directory, 'job.json') + : writeEmbeddingJobRecord(directory, 'job.json', { overwritten: true }), + ).rejects.toThrow('directory changed'); + expect(open).not.toHaveBeenCalled(); + expect(await readEmbeddingJobRecord(outside, 'job.json')).toEqual({ outside: true }); + }, +); + it.skipIf(process.platform === 'win32')( 'rejects a receipt replaced by a FIFO without blocking its open', async () => { diff --git a/gitnexus/test/unit/embedding-sidecar-client.test.ts b/gitnexus/test/unit/embedding-sidecar-client.test.ts index d417c74fb..9fad230f6 100644 --- a/gitnexus/test/unit/embedding-sidecar-client.test.ts +++ b/gitnexus/test/unit/embedding-sidecar-client.test.ts @@ -184,6 +184,30 @@ describe('embedding sidecar client', () => { } }); + it('pins model and chunk settings across environment changes before lazy initialization', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const lease = await reserveEmbeddingSidecar(config, 512); + const changed = { + MODEL_ID: 'different/model', + CHUNK_SIZE: '128', + CHUNK_OVERLAP: '32', + CHUNK_STRATEGY: 'fixed', + }; + try { + for (const [name, value] of Object.entries(changed)) + vi.stubEnv(`GITNEXUS_EMBEDDING_${name}`, value); + await lease.initialize(); + for (const name of Object.keys(changed)) + expect(forkMock.mock.calls[0][2].env?.[`GITNEXUS_EMBEDDING_${name}`]).toBeUndefined(); + expect(children[0].send.mock.calls[0][0]).toMatchObject({ embeddingConfig: config }); + } finally { + lease.release(); + vi.unstubAllEnvs(); + } + }); + it('waits for an idle old process to exit before assigning a different budget', async () => { const { reserveEmbeddingSidecar } = await import('../../src/core/embeddings/embedding-sidecar-client.js'); diff --git a/gitnexus/test/unit/watch-cancellation.test.ts b/gitnexus/test/unit/watch-cancellation.test.ts index 3e89dd531..ec6f98486 100644 --- a/gitnexus/test/unit/watch-cancellation.test.ts +++ b/gitnexus/test/unit/watch-cancellation.test.ts @@ -12,6 +12,7 @@ vi.mock('../../src/core/run-analyze.js', () => ({ })); vi.mock('../../src/core/analysis-host-resources.js', () => ({ analysisMayEmbed: mocks.mayEmbed, + analysisSupervisorReservation: () => 192 * 1024 ** 2, resolveHostAnalysisResources: () => ({ mainHeapMb: 512 }), })); import { watchCommandWithRunnerIdentity } from '../../src/cli/analyze-watch.js'; diff --git a/gitnexus/test/unit/watch-supervisor-reservation.test.ts b/gitnexus/test/unit/watch-supervisor-reservation.test.ts new file mode 100644 index 000000000..71abd5f9c --- /dev/null +++ b/gitnexus/test/unit/watch-supervisor-reservation.test.ts @@ -0,0 +1,62 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + resources: vi.fn((_request: { supervisorBytes?: number }) => ({ + mainHeapMb: 512, + supervisorBytes: 0, + })), + heap: vi.fn(async () => true), +})); +vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: mocks.heap })); +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: mocks.resources, +})); +import { + resolveWatchOptions, + watchCommandWithRunnerIdentity, +} from '../../src/cli/analyze-watch.js'; +import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; + +const directories: string[] = []; +afterEach(async () => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + await Promise.all( + directories.splice(0).map((directory) => fs.rm(directory, { recursive: true })), + ); +}); + +it.each([undefined, String(320 * 1024 ** 2)])( + 'accounts for the supervisor before heap respawn with inherited reservation %s', + async (inherited) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); + directories.push(directory); + await fs.mkdir(path.join(directory, '.git')); + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', inherited); + await watchCommandWithRunnerIdentity({} as AnalyzerRunnerIdentity, directory); + expect(mocks.resources).toHaveBeenCalledWith( + expect.objectContaining({ + supervisorBytes: inherited === undefined ? 192 * 1024 ** 2 : Number(inherited), + }), + ); + expect(mocks.heap).toHaveBeenCalledOnce(); + }, +); + +it('retains the parent reservation across watch refreshes', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); + directories.push(directory); + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', String(320 * 1024 ** 2)); + vi.stubEnv('GITNEXUS_RC', ''); + const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; + await resolveWatchOptions(directory, {}, baseline); + await resolveWatchOptions(directory, {}, baseline); + expect(mocks.resources).toHaveBeenCalledTimes(2); + for (const [request] of mocks.resources.mock.calls) + expect(request).toMatchObject({ supervisorBytes: 320 * 1024 ** 2 }); +}); diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index 3dad9e0e9..410077b25 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -127,6 +127,35 @@ describe('worker pool timeout retirement', () => { } }); + it('keeps an errored retired worker reserved until its physical exit', async () => { + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 20, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + worker.terminate = async () => { + worker.terminateCalls++; + throw new Error('termination rejected'); + }; + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.dispatch([{ path: 'src/stalled.ts', content: '' }])).rejects.toThrow(); + const retired = TimeoutThenHealthyWorker.instances[0]; + retired.emit('error', new Error('error before exit')); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + retired.emit('exit', 1); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + TimeoutThenHealthyWorker.instances[0]?.emit('exit', 1); + await pool.terminate().catch(() => {}); + } + }); + it('refuses replacement and phase handoff while a budgeted native worker has not drained', async () => { const pool = createWorkerPool(workerUrl, 1, { requireDrain: true, From edc091182262215284459cf6e2cb799e4851c09b Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 19:53:06 +0000 Subject: [PATCH 40/67] test: distinguish parameterized CI cases --- gitnexus/test/unit/analysis-host-resources.test.ts | 10 +++++++--- gitnexus/test/unit/embedding-streaming-text.test.ts | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts index f8ce5e2c3..acca9b25d 100644 --- a/gitnexus/test/unit/analysis-host-resources.test.ts +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -95,9 +95,13 @@ it('honors explicit heap-only overcommit but rejects a conflicting family ceilin ).toThrow('cannot admit'); }); -it.each([{ embeddingJob: {} }, { embeddingCheckpoint: {} }, { stats: { embeddings: 1 } }])( - 'reserves retry/forced model work from the exact branch metadata', - async (metadata) => { +it.each([ + ['pending job', { embeddingJob: {} }], + ['pending checkpoint', { embeddingCheckpoint: {} }], + ['preserved vectors', { stats: { embeddings: 1 } }], +])( + 'reserves retry/forced model work for %s from the exact branch metadata', + async (_case, metadata) => { mocks.loadMeta.mockResolvedValue(metadata); expect(await analysisMayEmbed('/repo', { force: true, branch: 'release' })).toBe(true); expect(mocks.placement).toHaveBeenCalledWith('/repo', 'release'); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts index d7c366c15..27256e370 100644 --- a/gitnexus/test/unit/embedding-streaming-text.test.ts +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -141,7 +141,7 @@ describe('bounded embedding text preserves existing vectors', () => { expect(release).toHaveBeenCalledOnce(); }); it.each(['short', 'x'.repeat(2401)])( - 'rejects a negative overlap without skipping source text', + 'rejects a negative overlap without skipping source text (case %#)', (content) => { expect(() => [...iterateCharacterChunks(content, 1, 1, 1200, -1)]).toThrow('overlap'); }, From 1dc68da0d480a04b41e8927b469825ed892328fe Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 19:55:54 +0000 Subject: [PATCH 41/67] docs: remove implementation plan from PR --- ...plan-resource-bounded-resumable-analyze.md | 2498 ----------------- 1 file changed, 2498 deletions(-) delete mode 100644 docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md diff --git a/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md b/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md deleted file mode 100644 index 714f12193..000000000 --- a/docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md +++ /dev/null @@ -1,2498 +0,0 @@ -# GitNexus Engineering Plan — resource-bounded, resumable analyze - -> Task: finish [GitNexus #3598](https://github.com/abhigyanpatwari/GitNexus/issues/3598), without performance degradation. -> Deepen cycle 1; full form; impact depth 3. This revision reconciles completed work and changes two integration decisions. It does not claim feature completion. -> Evidence: HEAD 8c6b680f3071025189804ffebac2fd3bc8bf1232 on `feat/resource-bounded-resumable-analyze`. Build-current local index matches HEAD, schema-4 analyzer receipt and working-tree content. Connected MCP graph resources have a native engine mismatch; local backend queries work. PDG has partial/source-offset limitations (§5). -> Evidence provenance schema 2; global dirty digest 0a9c85780067d9afcd0764f307b60891e3cee927ee11eaeb5ec7826d10fd82cd; 86 sorted cited paths; only this generated plan path excluded. No production/test/config edits, builds or test executions in this Deepen pass. - -## 1. Objective - -Publish a verified graph and keyword index before optional embeddings finish. Bound queued content, expanded chunks and vector work by resources, preserve durable paid work across interruption, and report graph freshness separately from embedding readiness. Preserve existing text/chunk semantics, graph-only/no-op/incremental performance, throughput and query latency at comparable resources. - -This revision replaces the original unconditional private-graph immutability/rollback promise with the repository's existing staged versus in-place guarantees. It also replaces in-process native teardown on exit-only CLI/server paths with an actual process-exit boundary. Neither change relaxes the issue's requirement that **after verified graph publication, embedding failure leaves that new graph available**. - -## 2. Current Behaviour - -- [verified] Thirteen commits are present, including the original plan. Twelve provide feasibility evidence and implementation pieces (§7). The common bounded controller still has **no production caller**. Analyze, embeddings sync and `/api/embed` still call `runEmbeddingPipeline` (`gitnexus/src/core/run-analyze.ts:4995`, `gitnexus/src/cli/embeddings-sync.ts:198`, `gitnexus/src/server/api.ts:2289`). The feature is not active end to end. -- [verified] `runFullAnalysis` locks the resolved slot, re-resolves it under the lock, calls its inner coordinator, then handles shared-store publication and releases the lock (`gitnexus/src/core/run-analyze.ts:1450–1575`). The new controller expects its caller to hold that lock. -- [verified] Private incremental writes normally mutate the existing graph. Atomic incremental copying is opt-in; shared commit graphs are immutable. The coordinator currently deletes `existingMeta.graphPath` after private-graph preparation, so it cannot simply accept a newly introduced private generation path without updated routing (`run-analyze.ts:2817–2840`). -- [verified] CLI and server workers set `skipNativeCloseOnExit: true`. The close choice is at `run-analyze.ts:5497–5500`. `closeLbugBeforeExit` checkpoints but leaves native handles alive and defers sidecar housekeeping (`gitnexus/src/core/lbug/lbug-adapter.ts:2521–2532`). Strict native release helpers do not make it safe to override this workaround. -- [verified] CLI skill generation consumes the live `pipelineResult` (`gitnexus/src/cli/analyze.ts:1697` onward). The worker IPC projection deliberately excludes it because serializing the KnowledgeGraph materializes corpus-sized arrays and loses behavior (`gitnexus/src/server/analyze-worker-ipc.ts:1–94`). -- [verified] `runPublishedEmbeddingJob` validates graph receipts and resource reservations, persists a graph-bound job, lazily opens the source/provider, and publishes only after native close. Its reuse source is currently only `metadata.vectorGeneration`; it does not reuse a different interrupted job even though its log promises retained compatible work (`gitnexus/src/core/embeddings/embedding-job.ts:116–330`). -- [verified] The staged publication helper requires an owned staging filename and writes a distinct generation file; it has no explicit in-place mode (`gitnexus/src/core/analysis-publication.ts:116–167,262–303`). The current vector selector checks generation matching but lacks an explicit `incrementalInProgress`/collapse gate (`gitnexus/src/storage/vector-generation.ts:33–81`). -- [verified] Optional resource plumbing, native/session drain, sidecar reservations, readiness consumers and progress types are implemented. The total-budget CLI flag, parser-side model drain, producer/IPC phase details and coherent production activation remain unfinished. Existing focused checks passed; the full candidate suite, whole-command benchmark matrix and final review have not run. - -## 3. Relevant Architecture - -[verified] The existing index lock and native-safe worker lifecycle are the coordination primitives. `runFullAnalysis` already rechecks storage/branch placement after waiting. The auto-sync worker launcher already waits for actual process exit before settlement. Server launch handling checks collapsed graphs before backend publication; that ordering must survive the phase split. - -[inferred] Keeping every private graph physically immutable would add an O(database-size) copy to tiny incremental updates on filesystems without copy-on-write. Logical generation receipts plus the existing in-place dirty-recovery contract avoid that added cost. The active embedding job holds the index lock, so its graph stays stable for its lifetime without requiring immutable files forever. - -[inferred] A thin supervisor is needed only where a graph worker must exit before the embedding phase. It must remain small, count against the total budget and reuse existing CLI/server process machinery. Graph-only and warm no-op invocations must not gain an unconditional extra fork or load model/vector modules. - -[verified] Separate vector artifacts already have bounded native pools and complete per-node receipts. Published artifacts are immutable to writers. A graph-bound scheduling journal and reusable paid vector data have different lifetimes; coupling both to the job identity prevents efficient rebasing after source changes. - -## 4. GitNexus Findings - -Freshness was established with exact equality between the MCP context's schema-4 runner receipt and local `status --json` index receipt, matching HEAD, `up-to-date`, empty incomplete reasons and current content. Local queries use the already built analyzer and isolated navigation store. No refresh/build was necessary in this Deepen pass. Connected `clusters` and `processes` resource reads failed because that server reads storage version 42 while the local index uses 47; local impact summaries supply bounded process context instead. - -| Current local call: `impact(target, upstream, maxDepth:3)` | Result and direct dependents | Treatment | -| --- | --- | --- | -| `runFullAnalysisInner`, file-scoped | LOW; `runFullAnalysis` | Preserve outer lock, shared-store, registry and cache sequencing. The old plan's benchmark `gc` false positives are retired. | -| `runPublishedEmbeddingJob`, file-scoped | UNKNOWN: “No callers resolved” | Source search confirms no production call; controller tests exist. UNKNOWN is not a safety clearance. | -| `decideEmbeddingResume`, file-scoped | CRITICAL; `runFullAnalysisInner`, `embeddingsSyncCommand`, `/api/embed` handler | Migrate all three callers and test identical force/drop/resume decisions. | -| `semanticSearch`, local-backend file | HIGH; `LocalBackend._queryImpl` | Preserve hybrid dispatch, ranking and graph hydration; test readiness gates. | -| `ensureHeap`, CLI analyze file | LOW; `analyzeCommand`, `watchCommandWithRunnerIdentity` | Preserve explicit heap, environment and watch behavior; avoid nested supervisors. | -| `runWorkerAnalysis`, worker-core file | LOW; message handler in `server/analyze-worker.ts` | Add nonterminal phase messages while retaining one terminal slot and exit-only cleanup. | -| `createLaunchAnalysisWorker`, launcher file | LOW; `createServer` | Delay embedding admission until actual exit; keep collapse/branch/cancel behavior. | - -[graph] Process summaries reach CLI/watch, API and query flows; they are sampled navigation, not exhaustive dependency proofs. [verified] Dynamic-import legacy embedding callers were checked with text search. Previously CRITICAL native-close/session edits remain high-risk seams; rerun their impacts before any further edit rather than carry old caller counts forward. - -## 5. Statement-Level PDG Findings - -[graph] A PDG layer exists. File-scoped `controls` probes returned truncated sets. Narrowing `impact(mode:"pdg", target:, line:5498)` to the executable graph-close statement returned `pdg-no-block-at-line`, UNKNOWN. The controller's line-211 slice returned 14 statements, was still truncated, and located `loadMetadata` at line 125 while current source has it at line 127. The initial line-5500 probe was a ternary continuation and was corrected before drawing this conclusion. - -These results do **not** establish a complete or correctly aligned handoff slice. No fabricated dependence edges enter the context pack. This is a statement-layer limitation despite current source/build/index provenance; unrelated PDG repair is outside the task. - -Source-verified constraints that govern implementation: - -1. Checkpoint success is not physical native release. Exit-only graph workers must actually exit before another phase consumes their reserved memory or reopens the graph. -2. Recheck cancellation after waiting for native/session release and before metadata/provider work (`embedding-job.ts:120–127`). -3. In-place mutation invalidates the published graph receipt before the first write; successful verification and durability precede replacement freshness metadata. -4. Graph publication, registry/cache/shared-store finalization and a readable new graph precede embedding admission. Graph-ready is not command completion. -5. Acknowledged units require complete chunk groups and native durability. Model calls cannot be inferred reusable from one row hash. -6. Repeated metadata rewrites, whole-graph copies, job-history scans, repeated HNSW construction and unbounded IPC are performance hazards to exclude and measure. - -## 6. Proposed Changes - -The following are implementation decisions, not descriptions of already activated behavior. - -### A. Logical graph generations with explicit publication modes - -Extend `prepareAnalysisPublication`/`publishPreparedAnalysis`, `runFullAnalysisInner`, graph routing and reader gates with explicit **staged** and **in-place** modes. - -- Staged full builds retain owned generation files and the existing prepared receipt/metadata recovery. Before publication, the previous graph remains available where the existing staged path provides that guarantee. -- Default private incremental writes retain the existing writable graph file and delta write plan. Before mutation, durably mark the attempt/dirty state and invalidate its old generation readiness. After graph/FTS verification, checkpoint and native release, publish a new logical generation bound to covered content, runner/parser/schema identity and that owned path. Do not copy or hash the entire graph file just to advance a generation. -- An interrupted in-place graph write retains the existing dirty/incomplete status and forced-rebuild recovery; it cannot promise the previous graph survived. Do not advertise graph readiness until verification and publication finish. This corrects the original plan's unconditional pre-publication rollback promise. -- A prepared receipt binds its attempt to the expected prior metadata/slot. Revalidate it under the index lock before completion; a later writer superseding the attempt makes it unusable. Do not trust timestamps alone. Pin prepared artifacts against cleanup until completed or safely superseded. -- Shared commit graphs remain physically immutable. After a private slot adopts the new format, retain its mutable graph under an owned noncanonical filename and reuse that same file across logical generations. Retire the canonical legacy name only after native release, using the existing verified publication/retirement mechanism without a full file copy. The physical filename need not encode every new logical generation ID. Legacy no-job slots stay on their current path. Private paths must not be mistaken for shared pointers or stripped unconditionally. Add dirty/collapse gating before any generation-aware vector query. - -The stable-graph guarantee for embedding is the published receipt **plus the held index lock**. After publication, all embedding failures preserve the new graph/FTS and graph freshness. Preserve existing FTS opt-out/degraded reporting, collapse checks, content retention and cache publication rules. - -### B. Native process exit as the CLI/server handoff - -Add a small proposed `core/analysis-phase-supervisor.ts` and adapt the existing CLI respawn and server/auto-sync worker launch paths. Do not introduce a general scheduler or a borrowed-lock/token protocol. - -1. Use a filesystem-only preflight to keep verified no-op/graph-only cases on their current fast path. A compatible pending job takes embedding-only resume without parsing or rewriting the graph. Recheck all decisions after acquiring the existing slot lock. -2. For actual graph work followed by embeddings on exit-only paths, run the graph phase in its own worker process. Preserve `skipNativeCloseOnExit`; checkpoint and prepare the durable publication receipt. Resolve the final repository name under the existing registry collision rules before CLI operations that need it. Complete live-KnowledgeGraph work in that child, including skill generation, with existing `--index-only` and self-commit protections. Any early registry name registration must still point to an old verified graph or an explicitly incomplete/prepared slot; it cannot publish new graph readiness. Transfer only a bounded allowlisted phase/result receipt, never `pipelineResult` or source maps. -3. Treat the graph-prepared message as nonterminal. Wait for actual child exit. Only then may the supervisor reconcile sidecars, reacquire the ordinary index lock, re-resolve the slot and validate the expected receipt/generation before finishing filesystem publication, registry freshness/shared-store/cache bookkeeping and finalization checks. Revalidate the child's registered name as well as slot ownership; a changed registration must not leave generated CLI context naming a different repository. If another writer superseded the attempt, stop with a clear retry reason before any paid request; do not publish stale metadata. -4. Once the graph is verified readable, refresh server graph readers and emit graph-ready while the command/SSE job stays active. Start the embedding continuation under its own admitted allocations. Keep the supervisor and model sidecar inside the same process-family memory budget. Reuse an existing heap-respawn parent where possible; do not stack supervisors. -5. Cooperative cancellation retains artifacts and waits for native-safe exit. A child that sent prepared/complete but has not exited still owns its resources. Never terminate a native-active thread to enforce a budget. Exactly one final event reflects success, failure or cancellation. Defer the CLI success banner and opt-in self-commit until the final outcome, preserving the pre-run edit-safety snapshot through a bounded projection. Graph-ready must not commit context files or print overall success. - -For genuinely in-process long-lived callers that already perform safe real close, retain `drainAnalysisNativeResources` and strict physical release accounting. Do not route exit-only callers through that helper as a substitute for process exit. `ensureHeap` sometimes returns without spawning, so ambient heap/opt-out paths need explicit coverage. - -### C. Rebase pending paid artifacts without copying the graph - -Separate the graph-bound **job identity** (manifest, selection, cursor, source/analyzer/schema) from the ownership identity of its **unpublished vector artifact**. Extend `runPublishedEmbeddingJob`, `VectorStore`, `EmbeddingJobJournal`, job metadata and vector publication together. - -An unchanged compatible job resumes its persisted manifest/cursor. A source/graph change creates a fresh graph-bound manifest and journal while adopting the same exclusively owned pending vector artifact through a durable rebase receipt. Validate vector-space and text identities first. For each current node, reuse only a complete matching input/chunk receipt; changed groups are recomputed, deleted/unselected groups are removed before final coverage publication. Do not reinterpret an old journal's completed cursor against a new manifest. - -Locate at most one active pending artifact per compatible vector/input space through a directly keyed owned receipt, so returning to a retained space needs no linear job-history scan. Retain at most one previous completed artifact as a read-only fallback while the pending writer accumulates reused/new groups. Repeated source edits reuse that same pending store, so work from the first interrupted attempt survives subsequent rebases without an ever-growing chain of prior jobs or native pools. Published stores are never reopened for writing; the first write after completed publication creates a new pending artifact and performs bounded lookups against the completed source. - -Rebase intent/adoption must be recoverable before mutation. A crash between metadata, ownership receipt and journal changes preserves the old paid artifact, proves one active owner and either finishes or rejects the intended rebase. Incompatible model/dimensions/text identity starts separate owned work with a reason; cleanup must not discard retained paid evidence prematurely. Scheduling/resource changes do not change unit or vector identity. Build HNSW once after current selected coverage is complete, checkpoint and close, then publish the pointer. - -### D. Preserve existing vectors without claiming fresh complete coverage - -Plain `analyze` must still preserve vectors without requesting new/changed-node embeddings. Keep the legacy no-job fast path lazy and free of migration/model/spool initialization. - -For a previously completed separate vector artifact after a graph update, keep an explicit **preserved historical coverage** reference. It may retain existing semantic-search behavior by hydrating returned IDs against the newly published graph and omitting deleted IDs. Report its source generation and `coverage: preserved` separately from native index availability; never call it complete for the current selection. Do not rewrite all vectors or invoke a provider merely to change this small reference. This is a compatibility exception for a previously completed artifact, not permission to expose an active/failed writer. While a new job is pending/failed, gate the vector lane before model initialization or vector-table access and continue graph/keyword queries. - -Legacy monolithic indexes remain readable. A write-triggered migration uses bounded complete-group validation/spill and retains the source until publication succeeds. Preserve recorded checkpoint/model/input provenance; never fill missing historical fields with the current configuration and claim they were verified. Relocation alone does not justify re-embedding a provably compatible group. When an explicit new-generation request lacks the provenance required to establish compatibility, retain the artifact, explain the exact unknown identity before requests and treat those groups as unverified rather than silently reusable. Plain preserve-only analysis still performs zero provider requests in that case. Include the additional first-generation work in migration/request-count evidence. - -Keep explicit force/drop semantics and existing legacy interrupted-checkpoint validation. Do not change graph schema identity solely for vector storage. An older binary must not mistake an obsolete canonical graph for current data; document rebuild/downgrade behavior and preserve rollback artifacts under owned names. - -### E. Budget admission and production integration - -Expose `--resource-budget ` for the process family; keep `--memory-budget` as the existing main-heap override. Resolve once and pass immutable allocations through CLI, watch, server and sync. Include supervisor RSS, graph/embedding native pools, worker heaps/native allocations, sidecar/model memory, encoded HTTP responses, queues and headroom. Never count thread RSS as separate process RSS. - -Wire `drainEmbeddingSidecarForAnalysis` before parser admission on actual graph work, not on a verified no-op. Preserve physical-release accounting after cancellation and failed closes. Automatic defaults are provisional until matched-resource measurements validate useful concurrency and total memory. Keep bounded manifests/chunk expansion, whole-manifest ID validation, File fallback rules and prepared/batched vector writes already implemented. - -Activate all three writers together only after the process/publication/rebase/readiness contracts are covered. No production embedding path may retain the corpus-size all-content reader. Update shared-store seeding and cleanup so prepared publications, pending jobs and live readers pin the correct artifacts; siblings cannot mutate or collect them. - -### F. Truthful phase progress across every boundary - -Carry the existing optional detail fields through actual producers, worker IPC, server jobs, CLI, MCP context, HTTP projection and SSE. Define completed-new + reused + pending + failed = total, with active a subset of pending; unknown totals remain unknown until selection is ready. Advance completion only after durable acknowledgement. Report model download separately. Graph-ready is nonterminal; preserved coverage is not current embedding completion. Public projection copies only enumerated/numeric safe fields, never filesystem paths, ownership tokens or internal identity payloads. Preserve existing response shapes with additive fields and verify all consumers. - -## 7. Implementation Sequence - -**Already completed at this HEAD — do not repeat:** - -| Commit | Completed result | -| --- | --- | -| `f0f3b9abe` | Original approved plan; superseded only by this Deepen revision. | -| `60ddccefa` | Native storage feasibility and baseline microbenchmarks; whole-command harness still missing. | -| `dbf62eab1` | Pure resource resolver and deterministic bounded manifest. | -| `ec9063f65` | Vector store and durable segmented journal. | -| `ba23148ba` | Bounded runner/text streaming and complete-group reuse. | -| `084c52895` | Staged publication and generation-aware readers; in-place mode still needed. | -| `f2fc492cf` | Published-job controller; no production callers yet. | -| `8bdd572fd` | Optional parser/native resource allocation plumbing. | -| `8dec4f273` | Physical sidecar reservations; parser admission not wired. | -| `700d6aca8` | Readiness/progress consumers; producer/IPC wiring incomplete. | -| `2363a1060` | Interrupted native bootstrap and journal recovery. | -| `f953c569e` | Verified explicit native pool release and retry accounting. | -| `8c6b680f3` | Main graph/session drain before in-process phase admission. | - -Steps 1–5 of the original plan have helper implementations and microbenchmarks. Original step 6 is partial: its consumers/helpers exist, but the new writer path is not activated. Original steps 7–8 remain open. The original end-to-end `measure.mjs` was never created; move that missing work to the first remaining step. - -**Remaining sequence, executable from this HEAD after the revised plan gate:** - -1. Create the paired whole-command benchmark harness and record the unchanged pinned baseline/environment. Use the existing microbenchmarks only as diagnostics. Capture no-op/tiny-incremental and process-startup cost early so the supervisor/publication choices are measured before activation. -2. Extend publication to logical in-place generations and implement the bounded native-exit handoff as inert adapters. Cover crash/slot-race/cancellation and CLI live-graph postprocessing. Keep staged and shared immutability where it already applies. -3. Implement pending-artifact rebase and preserved/legacy coverage policy. Test repeated graph changes between crashes and known-identity zero-request migration. Keep published stores immutable and old artifacts until verified adoption/publication. -4. Complete resource admission, shared-store pinning, progress/IPC and reader coverage gates. Validate all public shapes and no-op laziness. -5. Switch analyze, sync and `/api/embed` coherently to the common bounded runner, including embedding-only resume. Remove all unbounded production callers. Run actual CLI/server phase and reader tests, not only injected controller tests. -6. Run the full regression and performance matrix; tune any repeatable regression before shipment. Update RUNBOOK/CLI and benchmark documentation. Regenerate intentional goldens once at the end; do not replace baselines to hide regressions. -7. Run final graph change analysis and `gitnexus-review` on the implementation branch, plus a separate local surface if needed. Report unresolved findings and platform gaps. No push or PR is authorized by this plan. - -Each implementation commit requires impact before symbol edits, relevant focused checks and complete `detect_changes` before commit. Partial/truncated change analysis is unresolved. Preserve the existing isolated worktree and unrelated user edits. - -## 8. Test Strategy - -| Test file | Status and required scenarios | -| --- | --- | -| `gitnexus/test/unit/analysis-publication.test.ts` | **existing** — In-place mutation invalidates generation before writes; no full graph copy/hash on a tiny edit. Staged crash keeps old graph; in-place crash preserves existing dirty-recovery behavior; after publication embedding failures preserve new graph. | -| `gitnexus/test/integration/analyze-phase-publication.test.ts` | **new** — Graph child sends prepared then stays alive: zero provider/open-new-pool calls until exit. Cold/warm MCP and HTTP readers see new graph/FTS while separate vector writer is open. Crash before/after file and metadata publication; competing writer changes slot/generation during handoff: reject stale receipt without paid requests. Registry-name collision/change and --skills/self-commit: context names match final registration; prepared slot never appears graph-ready. | -| `gitnexus/test/unit/analyze-worker-core.test.ts` | **existing** — Graph-prepared is nonterminal; cancellation owns terminal slot; preserve skipNativeCloseOnExit. | -| `gitnexus/test/unit/analyze-worker-ipc.test.ts` | **existing** — Graph getters throw if traversed: bounded phase receipt still serializes; safe counters only. | -| `gitnexus/test/unit/analyze-launch-collapse.test.ts` | **existing** — Collapse rejected before backend publication; no embedding child or model admission. | -| `gitnexus/test/unit/analyze-launch-branch-settle.test.ts` | **existing** — External/shared/branch slot from result is revalidated after child exit. | -| `gitnexus/test/unit/auto-sync-analysis-worker.test.ts` | **existing** — Prepared IPC is not terminal; actual exit gates completion; cancelled native child is not force-killed. | -| `gitnexus/test/integration/embedding-job-controller.test.ts` | **existing** — Unchanged graph skips parsing and complete provider requests; graph changes rebase pending artifact. Two source changes between interruptions retain first-attempt paid groups and previous completed fallback. New model/text/dimensions reject incompatible reuse before requests; smaller resource budget does not invalidate units. | -| `gitnexus/test/integration/embedding-vector-store.test.ts` | **existing** — Current manifest validates complete groups; stale/deleted nodes pruned before publish; published store never mutated. Rebase crash at intent/adoption/checkpoint boundaries preserves owned paid artifact and rejects foreign identity. | -| `gitnexus/test/unit/embedding-job-journal.test.ts` | **existing** — One active unit; corrupted checksum/missing segment preserved and recovered only from complete native groups. | -| `gitnexus/test/integration/embedding-job-restart.test.ts` | **new** — Real child crashes at JS-visible intent/native-write/checkpoint/cursor/HNSW/pointer boundaries; at most active unit replay. ENOSPC, cancellation and repeated interruption keep published graph available and no duplicate/missing chunk rows. | -| `gitnexus/test/integration/embedding-resource-bounds.test.ts` | **new** — Lazy 10k/100k/1m descriptors and oversized Unicode nodes: fixed queue high-water mark, identical text/chunks, no corpus materialization. Smaller real-native case counts parent, graph/embedding child and sidecar peaks without double-counting thread RSS. | -| `gitnexus/test/unit/vector-generation.test.ts` | **existing** — Dirty/collapsed/mismatched graphs gate before native/model access; historical completed artifact is labeled preserved; pending job disables vector lane. | -| `gitnexus/test/unit/local-backend-vector-generation.test.ts` | **existing** — Preserved results hydrate current IDs and reject deleted nodes; active vector writer never opened; query ranking parity. | -| `gitnexus/test/unit/embedding-job-progress.test.ts` | **existing** — Completed+reused+pending+failed=total; active subset of pending; preserved is not current-complete; no false terminal event. | -| `gitnexus/test/unit/server-sse-payload.test.ts` | **existing** — Forward producer/IPC phase details through safe allowlist; graph-ready nonterminal; exactly one final success/failure/cancel event. | -| `gitnexus/test/unit/analyze-memory-budget.test.ts` | **existing** — Heap-only flag retains old meaning, including explicit ambient heap and respawn opt-out. | -| `gitnexus/test/unit/analysis-resource-budget.test.ts` | **existing** — Supervisor+children+native pools+sidecar+queues fit; defaults preserve useful concurrency when feasible. | -| `gitnexus/test/unit/lbug-analysis-resources.test.ts` | **existing** — Failed close remains charged; session drain blocks model/parser; cancellation during drain makes no metadata/provider calls. | -| `gitnexus/test/integration/embedding-native-release.test.ts` | **existing** — Explicit native pools are not reused before verified physical close; failed close retries, result-close failures clear active admission. | -| `gitnexus/test/unit/embedding-sidecar-client.test.ts` | **existing** — Parser phase waits for prior model-child exit; cancellation does not release a physically active request early. | -| `gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts` | **existing** — Keep legacy complete-group reuse, second-crash, changed/deleted nodes, force/drop behavior and known-identity relocation without provider requests. | -| `gitnexus/test/integration/vector-generation-compatibility.test.ts` | **new** — Plain analyze preserves legacy and previously completed vector coverage without automatic requests. Unknown legacy provenance remains unknown; explicit generation logs any unsafe-reuse invalidation; relocation alone never invalidates verified groups. Windows open handles, no VECTOR/FTS, shared sibling and external/branch storage, concurrent writers. | -| `gitnexus/test/integration/shared-store-embedding-reuse.test.ts` | **existing** — Prepared publication/pending vector work pins its artifacts; sibling cannot clean or mutate them. | - -The existing tests listed above were located; entries marked **new** are proposed and must be created before naming them in a runnable command. Extend focused commands with those tests once implemented. Existing scripts were verified in `gitnexus/package.json`: - -```sh -cd gitnexus && npm run build -cd gitnexus && npx tsc --noEmit -cd gitnexus && npm run typecheck:tests -cd gitnexus && npm test -- test/unit/analysis-publication.test.ts test/unit/vector-generation.test.ts test/unit/analyze-worker-core.test.ts test/unit/analyze-worker-ipc.test.ts test/unit/embedding-job-journal.test.ts test/unit/analysis-resource-budget.test.ts test/unit/lbug-analysis-resources.test.ts test/unit/server-sse-payload.test.ts -cd gitnexus && npm run test:integration -- test/integration/embedding-job-controller.test.ts test/integration/embedding-vector-store.test.ts test/integration/embedding-native-release.test.ts test/integration/analyze-staged-embedding-recovery.test.ts test/integration/shared-store-embedding-reuse.test.ts -cd gitnexus && npm test -cd gitnexus && node --import tsx bench/embedding-inputs/measure.cjs --check -cd gitnexus && node --import tsx bench/embedding-identities/measure.cjs --check -``` - -Also run ESLint and Prettier checks on changed files and any required package checks. Do not run the candidate performance matrix alongside indexing or the full test suite. Baseline full-suite evidence exists (30,685 passing, 91 skipped, four setup environment failures; the isolated setup rerun passed); it is not a candidate pass. The latest implementation commit has 80 focused passing tests across six suites plus build/type/lint checks. No tests were rerun during this planning-only pass. - -**Performance protocol:** use baseline `f641342d1fcbfdcb764c5e478db0f69e6cea899a` and candidate in isolated stores, same corpus/features, Node/native versions, provider/model/request limits and comparable resources. Alternate order, warm up, and collect at least seven paired samples; extend if noisy. Include cold graph-only, warm no-op, tiny incremental edit on non-CoW storage, preserved/all-cached vectors, fresh embeddings, local inference, fast deterministic HTTP, interrupted resume, legacy migration and concurrent warm/cold graph/keyword/vector queries. Include manifest creation, process handoff, native reopen, migration and final HNSW inside the timer. - -Measure total/phase time, throughput, query p50/p95, CPU, process-family peak RSS, queue bytes, native reservations, disk I/O and provider request counts. Verify equal text/chunk/output fingerprints and result ranking. Test default allocations separately from a constrained explicit budget, comparing like resources in each case. A cold query unable to read the new graph during embedding is a correctness failure regardless of speed. - -**No performance degradation is an acceptance gate, not an assumed result.** No arbitrary allowed slowdown. Existing native/vector/text microbenchmarks are promising diagnostics but do not establish whole-command non-regression. A seven-pair isolated HTTP response microbenchmark had a candidate/baseline geometric ratio around 1.019 and remains inconclusive; investigate it within the full provider path. Report raw samples and uncertainty, including migration/first-generation costs. Any repeatable regression requires tuning/design work before completion; inconclusive or unavailable measurements remain unresolved. - -## 9. Risk and Impact Analysis - -- **CRITICAL resume policy / HIGH query lane:** all current depth-1 dependents in §4 have explicit wiring and tests. Native close/session helpers were previously CRITICAL and require fresh impact before edits. The original proceed choice covered the original design; this structural revision returns to the LFG gate. -- **Native lifetime:** a terminal IPC message is not physical release. Keep the existing exit-only workaround and actual-exit accounting on every path, including cancellation and ambient-heap/no-respawn invocations. -- **Publication:** staged rollback and in-place dirty recovery are different guarantees. The post-publication embedding-failure guarantee applies to both. Never publish a stale handoff receipt after another writer changes the slot/generation. -- **Durable reuse:** job and artifact identity separation must not create two mutable owners or lose the oldest compatible paid groups after repeated rebases. Treat journal corruption/unknown ownership as uncertainty, not an empty job. -- **Compatibility:** preserved historical vectors may remain available under explicit coverage metadata, but active/failed new jobs cannot be advertised as complete. Unknown legacy provenance is never silently upgraded. -- **Performance:** full-DB copying, eager migration, extra no-op forks, unbounded IPC, repeated history scans, constrained default concurrency and extra HTTP processing can violate the user's request. Tests and paired measurements decide acceptance. -- **Scope:** only required CLI/core/storage/server/tests/bench/docs files. No language-provider changes, dependencies, CI/release/security configuration, production credentials, unrelated worktree changes or publishing actions. - -## 10. Files Expected to Change - -The following is the remaining implementation map. Existing helper code should be extended where needed, not re-created; only entries explicitly marked NEW are proposed files. Symbol names/ranges are grounded in the current investigation or unchanged verified source. Re-run impact and locate the precise edit range during work. - -| File | Remaining responsibility | -| --- | --- | -| `gitnexus/src/core/run-analyze.ts` | Integrate logical graph generations, deferred exit-only finalization and embedding-only resume; preserve existing no-op/in-place paths. | -| `gitnexus/src/core/analysis-publication.ts` | Add explicit staged/in-place publication modes, attempt binding and exit-verified recovery without whole-graph copies. | -| `gitnexus/src/core/analysis-phase-supervisor.ts` | NEW proposed filesystem/IPC phase supervisor; use actual child exit, existing lock acquisition and bounded result projection. | -| `gitnexus/src/cli/analyze.ts` | Compose supervision with heap respawn; keep live pipeline-result work in graph child and avoid extra forks for no-op/graph-only runs. | -| `gitnexus/src/cli/index.ts` | Expose --resource-budget and pass immutable allocations without changing --memory-budget. | -| `gitnexus/src/cli/analyze-watch.ts` | Preserve heap flags and safe lifecycle across repeated analyses. | -| `gitnexus/src/server/analyze-worker-core.ts` | Emit graph-prepared handoff separately from terminal completion; retain exit-only native cleanup. | -| `gitnexus/src/server/analyze-worker-ipc.ts` | Allowlist bounded phase receipts/counters; never serialize pipelineResult. | -| `gitnexus/src/server/analyze-launch.ts` | Wait for graph-worker exit, finish publication, refresh graph backend, then continue embeddings; exactly one terminal event. | -| `gitnexus/src/core/auto-sync/analysis-worker-launch.ts` | Preserve exit-based settlement and cooperative cancellation; carry nonterminal graph readiness. | -| `gitnexus/src/core/embeddings/embedding-job.ts` | Separate job identity from unpublished artifact ownership, reuse pending paid groups across rebases, and admit only after native release. | -| `gitnexus/src/core/embeddings/embedding-job-identity.ts` | Keep graph-bound scheduling identity separate from reusable input/vector-space identities. | -| `gitnexus/src/core/embeddings/embedding-job-runner.ts` | Validate/reuse complete groups in the current manifest, preserve durability-before-acknowledgement and bounded queues. | -| `gitnexus/src/core/lbug/vector-store.ts` | Allow an exclusively owned unpublished artifact to outlive one graph-bound journal; retain complete group proofs and immutable published stores. | -| `gitnexus/src/storage/embedding-job-journal.ts` | Crash-safe rebase receipt and fresh manifest/cursor; no in-memory history chain or loss of acknowledged compatible groups. | -| `gitnexus/src/storage/embedding-job.ts` | Owned filesystem artifacts and durable adoption/rebase records. | -| `gitnexus/src/core/embeddings/vector-publication.ts` | Verify current manifest coverage and close durability before publishing the artifact pointer. | -| `gitnexus/src/storage/repo-meta.ts` | Version optional graph mode, job/artifact references, preserved coverage and recorded legacy provenance without inventing missing identities. | -| `gitnexus/src/storage/vector-generation.ts` | Reject dirty/collapsed graphs; distinguish verified-current, preserved historical, legacy and pending states. | -| `gitnexus/src/storage/index-capabilities.ts` | Expose availability separately from current-generation coverage; preserve existing result shapes with additive fields. | -| `gitnexus/src/storage/embedding-job-progress.ts` | Represent preserved coverage and durable pending progress truthfully. | -| `gitnexus/src/core/embedding-checkpoint.ts` | Retain legacy proof/force/drop policy; share one caller decision for new jobs. | -| `gitnexus/src/cli/embeddings-sync.ts` | Route to the common bounded controller after ownership and graph validation. | -| `gitnexus/src/server/api.ts` | Route /api/embed through the controller and carry safe readiness/progress fields. | -| `gitnexus/src/core/shared-store-analyze.ts` | Publish graph facts independently of checkout-owned vector debt; preserve shared immutable graph ownership. | -| `gitnexus/src/storage/shared-store.ts` | Resolve explicit graph mode and pin artifacts referenced by prepared publication, active job or readers. | -| `gitnexus/src/storage/repo-manager.ts` | Preserve owned generation references across metadata and registry updates. | -| `gitnexus/src/core/analysis-resource-budget.ts` | Include supervisor and child process peaks; tune automatic allocations using matched-resource measurements. | -| `gitnexus/src/core/lbug/lbug-adapter.ts` | Keep strict admission for genuinely in-process callers; never use it to override exit-only graph cleanup. | -| `gitnexus/src/core/embeddings/embedding-sidecar-client.ts` | Wire prior-model drain before parser admission, while keeping no-op analysis lazy. | -| `gitnexus/src/core/analysis-progress.ts` | Carry graph and embedding phase counters from actual durable producer events. | -| `gitnexus/src/core/embeddings/types.ts` | Maintain additive safe progress contracts. | -| `gitnexus/src/mcp/local/local-backend.ts` | Gate vector selection before model/native vector work; preserve BM25 parallelism and matching graph hydration. | -| `gitnexus/src/core/embeddings/vector-generation-search.ts` | Serve a completed preserved artifact with explicit coverage state; never serve the active writer. | -| `gitnexus/src/core/index-freshness.ts` | Keep vector debt separate; dirty in-place graph mutation remains unavailable/unverified. | -| `gitnexus/src/cli/status.ts` | Report graph freshness, embedding phase and preserved/current coverage independently. | -| `gitnexus/src/server/ops-snapshot.ts` | Allowlist public progress; omit paths, tokens and internal identities. | -| `gitnexus/src/server/analyze-job.ts` | Represent nonterminal graph readiness and one terminal outcome. | -| `gitnexus/src/server/repo-projection.ts` | Expose additive coverage/readiness fields. | -| `gitnexus/src/mcp/resources.ts` | Expose the same graph/vector state in repository context. | -| `gitnexus/bench/analyze-resume/measure.mjs` | NEW proposed end-to-end paired harness; existing microbenchmarks do not replace it. | -| `gitnexus/bench/analyze-resume/README.md` | Document raw samples, environment, pair ordering, equal-work checks and limits. | -| `RUNBOOK.md` | Document phase recovery, resource flag, preserved coverage, migration and downgrade behavior. | - -Tests from §8 and existing benchmark artifacts are evidence; proposed harness/tests have absent-path provenance. Cleanup integrations may require adjacent owned-storage code already in the original scope, with impact and targeted verification first; unrelated refactors remain excluded. - -## 11. Reusable Implementation Context - -```json -{ - "implementation_context": { - "task_summary": "Finish #3598 from completed helper commits: preserve incremental performance, publish verified graph before a bounded durable embedding job, and validate no performance degradation.", - "acceptance_criteria": [ - "New verified graph and keyword queries work during/after embedding failure.", - "Retained content, expanded chunks, vectors and prefetch fit an explicit resource envelope independently of corpus size.", - "On compatible retry, acknowledged units cause no provider requests; at most the active unit replays.", - "All crash boundaries converge to complete unique per-node chunk sets.", - "Invalidation is selective and explained; graph and vector identities are separate.", - "CLI/MCP/HTTP report graph freshness, embedding progress and actual vector readiness independently.", - "Synthetic scaling tests and paired default/constrained performance measurements show no repeatable regression." - ], - "evidence_provenance": { - "schema_version": 2, - "head_commit": "8c6b680f3071025189804ffebac2fd3bc8bf1232", - "generated_plan_path": "docs/plans/2026-11-03-gitnexus-plan-resource-bounded-resumable-analyze.md", - "global_dirty_digest": { - "algorithm": "sha256", - "canonicalization": "gitnexus-evidence-provenance-v2 NUL-framed UTF-8 records", - "value": "0a9c85780067d9afcd0764f307b60891e3cee927ee11eaeb5ec7826d10fd82cd" - }, - "cited_path_manifest": [ - { - "path": "AGENTS.md", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", - "index_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", - "worktree_digest": "sha256:6525593affddf3aff06d2e7e9d855505a5f5135dd8cb6d5f67a552e1ac5cf912", - "untracked_digest": "absent" - }, - { - "path": "GUARDRAILS.md", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", - "index_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", - "worktree_digest": "sha256:e04523426887f8597b28c7796f3fc7deb188e74653e9b5b377b75cdb7ed3d472", - "untracked_digest": "absent" - }, - { - "path": "RUNBOOK.md", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", - "index_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", - "worktree_digest": "sha256:212110530525091c631dc9a3957bc08e0da5cd2bac2d30a21abb00739af72d34", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/README.md", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", - "index_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", - "worktree_digest": "sha256:b37524c4c64d1d0688533f5de0e2f894ddbda877bbc198c8a5407114a7c5c1f9", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/measure.mjs", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/native-baseline.json", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", - "index_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", - "worktree_digest": "sha256:b9960371de24ed894b3ccd98cb156a3ef4f1b0abe6b9ba57bd6cbe1ddd936fd7", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/native-primitives.mjs", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", - "index_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", - "worktree_digest": "sha256:3b0112c66e15f82c7cc8c4ac311952cbd8873b00ad0a517ece807f43a98b0ceb", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/text-streaming-baseline.json", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", - "index_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", - "worktree_digest": "sha256:f037a52275ad6f224a71a3f15bb6e93fd6ececedec5e5fcf62160dbcf7dea20b", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/text-streaming.mjs", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", - "index_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", - "worktree_digest": "sha256:2283b338d4a57e88d494d98a8bec5b1d7f0b119830ac80fbba2a309b4c46d5d8", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/vector-store-baseline.json", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", - "index_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", - "worktree_digest": "sha256:b967c251fc385dd596af8b6ca529cfc6f58056dd87fd1c03dcf3a33009add8e9", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/analyze-resume/vector-store.mjs", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", - "index_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", - "worktree_digest": "sha256:0c8d292fda4ae28ef5060ff7ff55f82b8aa880f75eacf41dfa630ee72261125b", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/embedding-identities/measure.cjs", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", - "index_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", - "worktree_digest": "sha256:7f8a9dd6bff3ddec82fc35565b4992e71a936ccf68c3e3e374391cf950099e14", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/bench/embedding-inputs/measure.cjs", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", - "index_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", - "worktree_digest": "sha256:c07314078597b01ee35f232ddd940f0ac0ab7678b1fdd7768d579d3d4ff8f22b", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/package.json", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", - "index_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", - "worktree_digest": "sha256:397c5d4c8566655d42b89d138a6d9f705e33014b348868631f4746df7a06edcf", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/cli/analyze-watch.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", - "index_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", - "worktree_digest": "sha256:94410e3e832339af291fc232e1b4bd99e5992a5acdfd610cb0528b6dc072ce0d", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/cli/analyze.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", - "index_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", - "worktree_digest": "sha256:f7ab0d38351c9c63c323016633be3093a1b0f2548e8474cb00b83de2d1420286", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/cli/embeddings-sync.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", - "index_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", - "worktree_digest": "sha256:3c62c4b91255422c481dfc27bf2e6fd216590ae922880d78a7812b45f46e02d6", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/cli/index.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", - "index_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", - "worktree_digest": "sha256:348cceda59baf473b6348b764435fd92bdea3bab045f94024991e2d22ab23acd", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/cli/status.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", - "index_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", - "worktree_digest": "sha256:1d17584ee2d7759f5b06d8d7e76dcc36f2658fd435aafadff935d6e89973bac0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/analysis-phase-supervisor.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/analysis-progress.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", - "index_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", - "worktree_digest": "sha256:d4e82adbf50d1c89d4e91b8d78abf510b1a55088fcc5d745c51bccae126b39d5", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/analysis-publication.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", - "index_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", - "worktree_digest": "sha256:eb884b1e6513cb54662b84a750a975007f9afd202609ea3a5e742f7353a8a471", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/analysis-resource-budget.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", - "index_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", - "worktree_digest": "sha256:7dc00edf90721d155eb5e9c961a859f79222dce3013dcb70afcf1ffc62491f3c", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", - "index_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", - "worktree_digest": "sha256:101a6395ccbc6de1ddbe1fe805ea863388115421dd37b77f0295e425988a2f04", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embedding-checkpoint.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", - "index_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", - "worktree_digest": "sha256:402a2501b355ea457f074802246f2de99e9ddf028502f5e874e74f411444ab08", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-job-identity.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", - "index_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", - "worktree_digest": "sha256:9549ef09c72478186e933f747c6d992f38e7cb0d79bc5438c69a93be1339b415", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-job-runner.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", - "index_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", - "worktree_digest": "sha256:1929322282068ac7baca7b06f76ab08db17f688005387c0a6bce6d57c5e157f9", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-job.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", - "index_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", - "worktree_digest": "sha256:abcf32e3b4d4d6af05610b14c0521fbb8b2e613c51391c3fdcf0dd4ff164d1ee", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-pipeline.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", - "index_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", - "worktree_digest": "sha256:0e6988332df01e3cb7f3302f91d1b43c9e3b42c1f04a2d229be496da6612403d", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-sidecar-client.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", - "index_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", - "worktree_digest": "sha256:686d195abe5d50c1bd3ab04d5fb03744cad8424656c9a60c17d39e93a23590c7", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/embedding-work-manifest.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", - "index_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", - "worktree_digest": "sha256:624eae51ded3820063890c65b75c56ce8ce705ea2d3364b9347b931e7bfcefc0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/types.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", - "index_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", - "worktree_digest": "sha256:4b41a883e9a5f62b0c6135696bae7880e1c247e8da8cc59f92fe8dfdff75ffb0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/vector-generation-search.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", - "index_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", - "worktree_digest": "sha256:c5efc5d48f40cff2579253a6368b3a6f9519d2fa9729b317c48898ee0a6dab5e", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/embeddings/vector-publication.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", - "index_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", - "worktree_digest": "sha256:74857e03e58d2e406604675db5370dac5f9e0d84c9c4e08b374963c2be78c968", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/index-freshness.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", - "index_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", - "worktree_digest": "sha256:395bd98aa5a286b23424930b93bf1d78807316d4810ebe964e19ff4a691f3056", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/ingestion/workers/worker-pool.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", - "index_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", - "worktree_digest": "sha256:941054f8291fb11f18c01f7e7b1644e15e994c1bd3ac7b03940e782815538d9c", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/lbug/embedding-source.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", - "index_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", - "worktree_digest": "sha256:e3aabd03c69aa74fa2a27b530d60654d71ea2e6734504b0ec4acbac0f6468737", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/lbug/lbug-adapter.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", - "index_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", - "worktree_digest": "sha256:828a6667a7b2bb832e6a063cf2ac7e7846727433eab46e72a62121975a6c29e5", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/lbug/lbug-config.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", - "index_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", - "worktree_digest": "sha256:83446f01048db52effb674415dfee16ae1a05acd8b0b1e6ce5d220be6fc802c6", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/lbug/pool-adapter.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", - "index_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", - "worktree_digest": "sha256:9166d33e94410bc05a28129a0159ef33872d67bf0b2f9ff0695c00bf8b1b6c3a", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/lbug/vector-store.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", - "index_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", - "worktree_digest": "sha256:eaf1ff71d135e4e384888277181dc3193f5664f94a7fde4daa13179293b68b4a", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/run-analyze.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", - "index_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", - "worktree_digest": "sha256:756030431ea263c658a29566877a63a6df40dbe138410e03c9af4713316ba722", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/core/shared-store-analyze.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", - "index_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", - "worktree_digest": "sha256:5aa5948076c18b603f1909b3d4c917aa692b9cf0f7ad1f0753b477f47a01d41e", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/mcp/local/local-backend.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", - "index_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", - "worktree_digest": "sha256:c1bfc2c305948dd77619c666a9f68420efee9d834c18be12a65a6d9d7f4683fa", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/mcp/resources.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", - "index_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", - "worktree_digest": "sha256:0689f5ca7b29320a9c919407d06551f80600c90a1596f832f03e25d6a050f7f6", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/analyze-job.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", - "index_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", - "worktree_digest": "sha256:013f35454d3f253fc1e1ddf1fd46f9b36f36a8579b17f4b6c9c22b3f342dd010", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/analyze-launch.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", - "index_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", - "worktree_digest": "sha256:886da3f19448f6ca01a504eee631aa7027c3c35ff27715e2ccfaa5bc35512b0f", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/analyze-worker-core.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", - "index_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", - "worktree_digest": "sha256:ff74b381cd1279f5a95dd6c964e375d03026bc102b102328af059e31ca7930f7", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/analyze-worker-ipc.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", - "index_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", - "worktree_digest": "sha256:80d6bba543ee757fa65ff036b42544aba6ac052d1f536eb407d16b1589470a10", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/analyze-worker.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", - "index_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", - "worktree_digest": "sha256:0cb4fa75441c31eb674102db7f15b95645a81c6cd7cdedac6ce0f6d5d74a389c", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/api.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", - "index_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", - "worktree_digest": "sha256:724c463aaf317c9b8d7e7b420472d66ba4838dc91d8a978aa4791d01993c4412", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/ops-snapshot.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", - "index_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", - "worktree_digest": "sha256:a89c2718afea565aa52fdb7ca577a9e9e81d57960bbe664f478f3a2c405c2365", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/server/repo-projection.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", - "index_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", - "worktree_digest": "sha256:779fa65dc8b6ff7cd77c449938f3d4a3058ea2497f1a03d70652c59b69a84589", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/embedding-job-journal.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", - "index_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", - "worktree_digest": "sha256:45b2d6d1db96e53a06e22cb13ca3debfb67e200b0eff5e99c3bad69efe643be0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/embedding-job-progress.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", - "index_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", - "worktree_digest": "sha256:e5d19ae5b416c115164e0988bd8b30e43e476fecede524d17126168fb69f9039", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/embedding-job.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", - "index_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", - "worktree_digest": "sha256:aa2a245ce1352e89bf1c5d0ac78ef15c9a33b4939269b9757a2574853599bd4a", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/index-capabilities.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", - "index_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", - "worktree_digest": "sha256:1106d95598245248a7f093d9701c4ab37c73e175f074d8a6561b1d871970600d", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/index-lock.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", - "index_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", - "worktree_digest": "sha256:2285d73a52f619e1c742865e7d280c9e94bc10e2d7749fa557c6f44491057c90", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/repo-manager.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", - "index_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", - "worktree_digest": "sha256:412e87ab75c7db23c323e5d04aaafb03351ee7f2118dc307ccc19cab01e4e120", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/repo-meta.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", - "index_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", - "worktree_digest": "sha256:48163d6460e6403a7837c25e7fff5209c7ca0d7f71e82f71499a87e0bc1c6c92", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/shared-store.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", - "index_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", - "worktree_digest": "sha256:be639f0f54680f27abcf1fc1efdfdadfbb2197d1ec4d493cdf03d6b58aebad17", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/src/storage/vector-generation.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", - "index_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", - "worktree_digest": "sha256:8e68833a8d0741ded37bd4b4600e2e094eb8d631f7862deb3be039cd23fa3655", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/analyze-phase-publication.test.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", - "index_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", - "worktree_digest": "sha256:68f572fc65126eab6ea033d28a22f9e1025746545181760e62cfce1adf12fd13", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/embedding-job-controller.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", - "index_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", - "worktree_digest": "sha256:052d885d773597196a719cd1184be4d3349c45c74836c992937ad75c10afff84", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/embedding-job-restart.test.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/embedding-native-release.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", - "index_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", - "worktree_digest": "sha256:43dcc9d5b5dbd61d07bc44bf1191f12c5b12f05aa32e9d384adb7d97effb9819", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/embedding-resource-bounds.test.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/embedding-vector-store.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", - "index_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", - "worktree_digest": "sha256:1b9819c2692e6122ba9f958ca07acde122005c53723af7bb17961fb3323b5f2a", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", - "index_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", - "worktree_digest": "sha256:33f8d89e66a8600c92965c08dada1dd658621b8aa8edccfa034a6ef8a3e30d80", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/integration/vector-generation-compatibility.test.ts", - "object_kind": { - "head": "absent", - "index": "absent", - "worktree": "absent", - "untracked": "absent" - }, - "state": "absent", - "rename_from": null, - "rename_to": null, - "head_digest": "absent", - "index_digest": "absent", - "worktree_digest": "absent", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analysis-publication.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", - "index_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", - "worktree_digest": "sha256:e37f3c7a85bc17dbc7a1ccb2303eeb6a70cb3be4b32d4c90a8882945cf8813e0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analysis-resource-budget.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", - "index_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", - "worktree_digest": "sha256:24ed33c78bb6f37abde2f6b68ba35c307c3d1c25fc2b14830407f48b6a109acf", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analyze-launch-branch-settle.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", - "index_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", - "worktree_digest": "sha256:9a37459f431b81dc89d2b02e5c05f361b41cd4ff598e02b27371b5fdfe2c0e1b", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analyze-launch-collapse.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", - "index_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", - "worktree_digest": "sha256:e1fd4983f39929aba6beb5ddd55b02ab31b85e26de4a040dd781658ee88fd652", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analyze-memory-budget.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", - "index_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", - "worktree_digest": "sha256:eef96d7af6d47b242c7a4d46848641a7a8975c7967b058177cce40f0673ec934", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analyze-worker-core.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", - "index_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", - "worktree_digest": "sha256:40a12110cecb3542e218d85d4aed7bd1fde6c168479947ed69b691a57d417bc0", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/analyze-worker-ipc.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", - "index_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", - "worktree_digest": "sha256:f57c5cce114929f78c37fd7b4ed32a4061d87f9bf712d7c2ddef65d061fad68e", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/auto-sync-analysis-worker.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", - "index_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", - "worktree_digest": "sha256:e6f69f81057968879092a6df90619de873a8fb9a0d1b192f4450f6ec72058950", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/embedding-job-journal.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", - "index_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", - "worktree_digest": "sha256:b77958ad828ec1392f5c53f6317c7a96176a8bbf1f0300e080b86326b570bcf6", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/embedding-job-progress.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", - "index_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", - "worktree_digest": "sha256:1fee3af2ce6ece7e89c2e87015d3e444ce766b9a5a5b194fd55aca82f9c9de0c", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/embedding-sidecar-client.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", - "index_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", - "worktree_digest": "sha256:ad2c7746c20179d797ff1fa2252f4824025a3b600f601146d9af754916159fc8", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/lbug-analysis-resources.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", - "index_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", - "worktree_digest": "sha256:a8f81cec6a18f4f8388599b7c16aec643ebe5bbb17c9c7458c5af2e584ad9e05", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/local-backend-vector-generation.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", - "index_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", - "worktree_digest": "sha256:62dfd503c616fa0b9f03737add16f84aa60f7bba6406390de925418a602d50c6", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/server-sse-payload.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", - "index_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", - "worktree_digest": "sha256:116cb2e25c2acb6bbbe789506ceecd2761b344a53269c634511240f49f8423a5", - "untracked_digest": "absent" - }, - { - "path": "gitnexus/test/unit/vector-generation.test.ts", - "object_kind": { - "head": "regular", - "index": "regular", - "worktree": "regular", - "untracked": "absent" - }, - "state": "clean", - "rename_from": null, - "rename_to": null, - "head_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", - "index_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", - "worktree_digest": "sha256:34709e135f248a242ffd3c5ca2607423df94da785bec99a2ec0208d000073be6", - "untracked_digest": "absent" - } - ] - }, - "primary_symbols": [ - { - "symbol": "runFullAnalysisInner", - "file": "gitnexus/src/core/run-analyze.ts", - "lines": "1580,2817-2840,5497-5500,5545-5580", - "role": "Graph finalization; retain in-place vs staged semantics and defer exit-only publication", - "source_verified": true - }, - { - "symbol": "runPublishedEmbeddingJob", - "file": "gitnexus/src/core/embeddings/embedding-job.ts", - "lines": "116-330", - "role": "Bounded controller, verified graph admission, pending-artifact rebase", - "source_verified": true - }, - { - "symbol": "decideEmbeddingResume", - "file": "gitnexus/src/core/embedding-checkpoint.ts", - "lines": "190-290", - "role": "Existing force/drop/legacy compatibility across three production callers", - "source_verified": true - }, - { - "symbol": "LocalBackend.semanticSearch", - "file": "gitnexus/src/mcp/local/local-backend.ts", - "lines": "generation selection and graph hydration; re-locate exact edit range before change", - "role": "Readiness and preserved coverage gating", - "source_verified": true - }, - { - "symbol": "ensureHeap", - "file": "gitnexus/src/cli/analyze.ts", - "lines": "682-764", - "role": "Existing conditional respawn; supervisor cannot assume a parent already exists", - "source_verified": true - } - ], - "related_symbols": [ - { - "symbol": "runFullAnalysis", - "relationship": "CALLS runFullAnalysisInner", - "relevance": "Lock, shared-store, cache and registry boundary" - }, - { - "symbol": "analyzeCommandImpl", - "relationship": "CALLS runFullAnalysis", - "relevance": "CLI postprocessing consumes live graph" - }, - { - "symbol": "runWorkerAnalysis", - "relationship": "CALLS runFullAnalysis", - "relevance": "skipNativeCloseOnExit plus terminal projection" - }, - { - "symbol": "createLaunchAnalysisWorker", - "relationship": "parent of graph worker", - "relevance": "Actual exit before continuation and backend publication" - }, - { - "symbol": "projectAnalyzeResultForIpc", - "relationship": "projection", - "relevance": "Bounded scalar allowlist" - }, - { - "symbol": "watchCommandWithRunnerIdentity", - "relationship": "CALLS ensureHeap/runFullAnalysis", - "relevance": "Repeated process lifecycle" - }, - { - "symbol": "embeddingsSyncCommand", - "relationship": "CALLS decideEmbeddingResume", - "relevance": "Production writer parity" - }, - { - "symbol": "createServer / POST /api/embed", - "relationship": "CALLS decideEmbeddingResume", - "relevance": "Production writer parity" - }, - { - "symbol": "LocalBackend._queryImpl", - "relationship": "CALLS semanticSearch", - "relevance": "Preserve hybrid search concurrency/ranking" - }, - { - "symbol": "prepareAnalysisPublication / publishPreparedAnalysis", - "relationship": "publication helper", - "relevance": "Extend staged-only receipt to explicit in-place mode" - }, - { - "symbol": "selectVectorGeneration", - "relationship": "read gate", - "relevance": "Dirty graph, pending writer and preserved coverage" - }, - { - "symbol": "VectorStore", - "relationship": "native storage", - "relevance": "Stable unpublished artifact plus complete group receipts" - }, - { - "symbol": "EmbeddingJobJournal", - "relationship": "durability", - "relevance": "Graph-specific manifest/cursor and crash-safe rebase" - }, - { - "symbol": "runEmbeddingJob", - "relationship": "runner", - "relevance": "Bounded complete-unit processing" - }, - { - "symbol": "drainAnalysisNativeResources", - "relationship": "resource admission", - "relevance": "Main singleton and tracked explicit pool release" - }, - { - "symbol": "drainEmbeddingSidecarForAnalysis", - "relationship": "resource admission", - "relevance": "Release prior runtime before parser phase" - }, - { - "symbol": "publishSharedGraph", - "relationship": "shared storage", - "relevance": "Immutable shared graph separate from checkout vector work" - }, - { - "symbol": "resolveGraphPath", - "relationship": "routing", - "relevance": "Owned canonical/generation/shared paths" - }, - { - "symbol": "getGraphIncompleteReasons", - "relationship": "status", - "relevance": "Graph debt separate from vectors" - }, - { - "symbol": "runEmbeddingPipeline", - "relationship": "legacy production path", - "relevance": "Three callers remain; must be removed from unbounded production use at activation" - } - ], - "execution_path": [ - "Lock and revalidate target; take existing no-op or embedding-only resume when valid.", - "If graph work is required, drain prior runtime and run bounded graph phase with its existing in-place/staged choice.", - "Checkpoint and prepare receipt; finish live-graph CLI work in graph child; on exit-only paths observe actual child exit.", - "Reacquire existing slot lock, revalidate attempt/slot/metadata, finalize sidecars and publication, registry/shared-store/cache; graph-ready is nonterminal.", - "Reuse/rebase one owned pending vector artifact; persist graph-specific manifest/journal; admit separate vector writer/model within budget.", - "Verify current selected coverage, checkpoint/close, publish vector receipt; report exactly one final outcome." - ], - "pdg_constraints": [], - "pdg_availability": "Layer exists. Bounded controls/impact probes are partial and source-offset inconsistent; no trusted complete slice for native handoff. Do not infer edges from source.", - "source_constraints": [ - "Exit-only CHECKPOINT does not release graph/native handles.", - "In-place mutation must invalidate the published graph receipt before the first write.", - "One index lock pins a graph during the active embedding job; a handoff gap requires revalidation.", - "Only complete per-node chunk receipts qualify for reuse; durability precedes cursor acknowledgement.", - "No model initialization or vector table access before readiness gate; no-op stays lazy.", - "Final registry name used by graph-dependent CLI outputs must match post-exit publication; early registration cannot advertise new graph readiness." - ], - "architectural_patterns": [ - { - "pattern": "Owned filesystem receipts and safe publication", - "example_location": "gitnexus/src/core/analysis-publication.ts", - "usage_guidance": "Extend explicit publication modes; do not weaken ownership or checksum checks." - }, - { - "pattern": "Bounded IPC projection", - "example_location": "gitnexus/src/server/analyze-worker-ipc.ts:projectAnalyzeResultForIpc", - "usage_guidance": "Keep KnowledgeGraph in child; allowlist only phase receipt and scalar counters." - }, - { - "pattern": "Actual exit before settlement", - "example_location": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", - "usage_guidance": "Reuse exit/cancellation discipline; never force-kill native work." - }, - { - "pattern": "Exclusive index lock and target revalidation", - "example_location": "gitnexus/src/core/run-analyze.ts:runFullAnalysis", - "usage_guidance": "At handoff reacquire existing lock and reject replaced receipt/slot/generation." - } - ], - "files_to_modify": [ - { - "file": "gitnexus/src/core/run-analyze.ts", - "symbols": [ - "runFullAnalysis", - "runFullAnalysisInner" - ], - "intended_change": "Integrate logical graph generations, deferred exit-only finalization and embedding-only resume; preserve existing no-op/in-place paths." - }, - { - "file": "gitnexus/src/core/analysis-publication.ts", - "symbols": [ - "prepareAnalysisPublication", - "publishPreparedAnalysis" - ], - "intended_change": "Add explicit staged/in-place publication modes, attempt binding and exit-verified recovery without whole-graph copies." - }, - { - "file": "gitnexus/src/core/analysis-phase-supervisor.ts", - "symbols": [], - "intended_change": "NEW proposed filesystem/IPC phase supervisor; use actual child exit, existing lock acquisition and bounded result projection." - }, - { - "file": "gitnexus/src/cli/analyze.ts", - "symbols": [ - "ensureHeap", - "analyzeCommandImpl" - ], - "intended_change": "Compose supervision with heap respawn; keep live pipeline-result work in graph child and avoid extra forks for no-op/graph-only runs." - }, - { - "file": "gitnexus/src/cli/index.ts", - "symbols": [], - "intended_change": "Expose --resource-budget and pass immutable allocations without changing --memory-budget." - }, - { - "file": "gitnexus/src/cli/analyze-watch.ts", - "symbols": [ - "watchCommandWithRunnerIdentity" - ], - "intended_change": "Preserve heap flags and safe lifecycle across repeated analyses." - }, - { - "file": "gitnexus/src/server/analyze-worker-core.ts", - "symbols": [ - "runWorkerAnalysis" - ], - "intended_change": "Emit graph-prepared handoff separately from terminal completion; retain exit-only native cleanup." - }, - { - "file": "gitnexus/src/server/analyze-worker-ipc.ts", - "symbols": [ - "projectAnalyzeResultForIpc" - ], - "intended_change": "Allowlist bounded phase receipts/counters; never serialize pipelineResult." - }, - { - "file": "gitnexus/src/server/analyze-launch.ts", - "symbols": [ - "createLaunchAnalysisWorker" - ], - "intended_change": "Wait for graph-worker exit, finish publication, refresh graph backend, then continue embeddings; exactly one terminal event." - }, - { - "file": "gitnexus/src/core/auto-sync/analysis-worker-launch.ts", - "symbols": [], - "intended_change": "Preserve exit-based settlement and cooperative cancellation; carry nonterminal graph readiness." - }, - { - "file": "gitnexus/src/core/embeddings/embedding-job.ts", - "symbols": [ - "runPublishedEmbeddingJob" - ], - "intended_change": "Separate job identity from unpublished artifact ownership, reuse pending paid groups across rebases, and admit only after native release." - }, - { - "file": "gitnexus/src/core/embeddings/embedding-job-identity.ts", - "symbols": [ - "embeddingJobIdentity" - ], - "intended_change": "Keep graph-bound scheduling identity separate from reusable input/vector-space identities." - }, - { - "file": "gitnexus/src/core/embeddings/embedding-job-runner.ts", - "symbols": [ - "runEmbeddingJob" - ], - "intended_change": "Validate/reuse complete groups in the current manifest, preserve durability-before-acknowledgement and bounded queues." - }, - { - "file": "gitnexus/src/core/lbug/vector-store.ts", - "symbols": [ - "VectorStore" - ], - "intended_change": "Allow an exclusively owned unpublished artifact to outlive one graph-bound journal; retain complete group proofs and immutable published stores." - }, - { - "file": "gitnexus/src/storage/embedding-job-journal.ts", - "symbols": [ - "EmbeddingJobJournal" - ], - "intended_change": "Crash-safe rebase receipt and fresh manifest/cursor; no in-memory history chain or loss of acknowledged compatible groups." - }, - { - "file": "gitnexus/src/storage/embedding-job.ts", - "symbols": [], - "intended_change": "Owned filesystem artifacts and durable adoption/rebase records." - }, - { - "file": "gitnexus/src/core/embeddings/vector-publication.ts", - "symbols": [ - "publishVectorGeneration" - ], - "intended_change": "Verify current manifest coverage and close durability before publishing the artifact pointer." - }, - { - "file": "gitnexus/src/storage/repo-meta.ts", - "symbols": [ - "RepoMeta" - ], - "intended_change": "Version optional graph mode, job/artifact references, preserved coverage and recorded legacy provenance without inventing missing identities." - }, - { - "file": "gitnexus/src/storage/vector-generation.ts", - "symbols": [ - "selectVectorGeneration", - "metadataMatchesGraph" - ], - "intended_change": "Reject dirty/collapsed graphs; distinguish verified-current, preserved historical, legacy and pending states." - }, - { - "file": "gitnexus/src/storage/index-capabilities.ts", - "symbols": [], - "intended_change": "Expose availability separately from current-generation coverage; preserve existing result shapes with additive fields." - }, - { - "file": "gitnexus/src/storage/embedding-job-progress.ts", - "symbols": [], - "intended_change": "Represent preserved coverage and durable pending progress truthfully." - }, - { - "file": "gitnexus/src/core/embedding-checkpoint.ts", - "symbols": [ - "decideEmbeddingResume" - ], - "intended_change": "Retain legacy proof/force/drop policy; share one caller decision for new jobs." - }, - { - "file": "gitnexus/src/cli/embeddings-sync.ts", - "symbols": [ - "embeddingsSyncCommand" - ], - "intended_change": "Route to the common bounded controller after ownership and graph validation." - }, - { - "file": "gitnexus/src/server/api.ts", - "symbols": [ - "createServer" - ], - "intended_change": "Route /api/embed through the controller and carry safe readiness/progress fields." - }, - { - "file": "gitnexus/src/core/shared-store-analyze.ts", - "symbols": [ - "publishSharedGraph" - ], - "intended_change": "Publish graph facts independently of checkout-owned vector debt; preserve shared immutable graph ownership." - }, - { - "file": "gitnexus/src/storage/shared-store.ts", - "symbols": [ - "resolveGraphPath" - ], - "intended_change": "Resolve explicit graph mode and pin artifacts referenced by prepared publication, active job or readers." - }, - { - "file": "gitnexus/src/storage/repo-manager.ts", - "symbols": [ - "saveMeta" - ], - "intended_change": "Preserve owned generation references across metadata and registry updates." - }, - { - "file": "gitnexus/src/core/analysis-resource-budget.ts", - "symbols": [], - "intended_change": "Include supervisor and child process peaks; tune automatic allocations using matched-resource measurements." - }, - { - "file": "gitnexus/src/core/lbug/lbug-adapter.ts", - "symbols": [ - "drainAnalysisNativeResources" - ], - "intended_change": "Keep strict admission for genuinely in-process callers; never use it to override exit-only graph cleanup." - }, - { - "file": "gitnexus/src/core/embeddings/embedding-sidecar-client.ts", - "symbols": [ - "drainEmbeddingSidecarForAnalysis" - ], - "intended_change": "Wire prior-model drain before parser admission, while keeping no-op analysis lazy." - }, - { - "file": "gitnexus/src/core/analysis-progress.ts", - "symbols": [], - "intended_change": "Carry graph and embedding phase counters from actual durable producer events." - }, - { - "file": "gitnexus/src/core/embeddings/types.ts", - "symbols": [ - "EmbeddingProgress" - ], - "intended_change": "Maintain additive safe progress contracts." - }, - { - "file": "gitnexus/src/mcp/local/local-backend.ts", - "symbols": [ - "LocalBackend.semanticSearch" - ], - "intended_change": "Gate vector selection before model/native vector work; preserve BM25 parallelism and matching graph hydration." - }, - { - "file": "gitnexus/src/core/embeddings/vector-generation-search.ts", - "symbols": [], - "intended_change": "Serve a completed preserved artifact with explicit coverage state; never serve the active writer." - }, - { - "file": "gitnexus/src/core/index-freshness.ts", - "symbols": [ - "getGraphIncompleteReasons" - ], - "intended_change": "Keep vector debt separate; dirty in-place graph mutation remains unavailable/unverified." - }, - { - "file": "gitnexus/src/cli/status.ts", - "symbols": [], - "intended_change": "Report graph freshness, embedding phase and preserved/current coverage independently." - }, - { - "file": "gitnexus/src/server/ops-snapshot.ts", - "symbols": [], - "intended_change": "Allowlist public progress; omit paths, tokens and internal identities." - }, - { - "file": "gitnexus/src/server/analyze-job.ts", - "symbols": [], - "intended_change": "Represent nonterminal graph readiness and one terminal outcome." - }, - { - "file": "gitnexus/src/server/repo-projection.ts", - "symbols": [], - "intended_change": "Expose additive coverage/readiness fields." - }, - { - "file": "gitnexus/src/mcp/resources.ts", - "symbols": [], - "intended_change": "Expose the same graph/vector state in repository context." - }, - { - "file": "gitnexus/bench/analyze-resume/measure.mjs", - "symbols": [], - "intended_change": "NEW proposed end-to-end paired harness; existing microbenchmarks do not replace it." - }, - { - "file": "gitnexus/bench/analyze-resume/README.md", - "symbols": [], - "intended_change": "Document raw samples, environment, pair ordering, equal-work checks and limits." - }, - { - "file": "RUNBOOK.md", - "symbols": [], - "intended_change": "Document phase recovery, resource flag, preserved coverage, migration and downgrade behavior." - } - ], - "tests": [ - { - "file": "gitnexus/test/unit/analysis-publication.test.ts", - "status": "existing", - "scenarios": [ - "In-place mutation invalidates generation before writes; no full graph copy/hash on a tiny edit.", - "Staged crash keeps old graph; in-place crash preserves existing dirty-recovery behavior; after publication embedding failures preserve new graph." - ] - }, - { - "file": "gitnexus/test/integration/analyze-phase-publication.test.ts", - "status": "new", - "scenarios": [ - "Graph child sends prepared then stays alive: zero provider/open-new-pool calls until exit.", - "Cold/warm MCP and HTTP readers see new graph/FTS while separate vector writer is open.", - "Crash before/after file and metadata publication; competing writer changes slot/generation during handoff: reject stale receipt without paid requests.", - "Registry-name collision/change and --skills/self-commit: context names match final registration; prepared slot never appears graph-ready." - ] - }, - { - "file": "gitnexus/test/unit/analyze-worker-core.test.ts", - "status": "existing", - "scenarios": [ - "Graph-prepared is nonterminal; cancellation owns terminal slot; preserve skipNativeCloseOnExit." - ] - }, - { - "file": "gitnexus/test/unit/analyze-worker-ipc.test.ts", - "status": "existing", - "scenarios": [ - "Graph getters throw if traversed: bounded phase receipt still serializes; safe counters only." - ] - }, - { - "file": "gitnexus/test/unit/analyze-launch-collapse.test.ts", - "status": "existing", - "scenarios": [ - "Collapse rejected before backend publication; no embedding child or model admission." - ] - }, - { - "file": "gitnexus/test/unit/analyze-launch-branch-settle.test.ts", - "status": "existing", - "scenarios": [ - "External/shared/branch slot from result is revalidated after child exit." - ] - }, - { - "file": "gitnexus/test/unit/auto-sync-analysis-worker.test.ts", - "status": "existing", - "scenarios": [ - "Prepared IPC is not terminal; actual exit gates completion; cancelled native child is not force-killed." - ] - }, - { - "file": "gitnexus/test/integration/embedding-job-controller.test.ts", - "status": "existing", - "scenarios": [ - "Unchanged graph skips parsing and complete provider requests; graph changes rebase pending artifact.", - "Two source changes between interruptions retain first-attempt paid groups and previous completed fallback.", - "New model/text/dimensions reject incompatible reuse before requests; smaller resource budget does not invalidate units." - ] - }, - { - "file": "gitnexus/test/integration/embedding-vector-store.test.ts", - "status": "existing", - "scenarios": [ - "Current manifest validates complete groups; stale/deleted nodes pruned before publish; published store never mutated.", - "Rebase crash at intent/adoption/checkpoint boundaries preserves owned paid artifact and rejects foreign identity." - ] - }, - { - "file": "gitnexus/test/unit/embedding-job-journal.test.ts", - "status": "existing", - "scenarios": [ - "One active unit; corrupted checksum/missing segment preserved and recovered only from complete native groups." - ] - }, - { - "file": "gitnexus/test/integration/embedding-job-restart.test.ts", - "status": "new", - "scenarios": [ - "Real child crashes at JS-visible intent/native-write/checkpoint/cursor/HNSW/pointer boundaries; at most active unit replay.", - "ENOSPC, cancellation and repeated interruption keep published graph available and no duplicate/missing chunk rows." - ] - }, - { - "file": "gitnexus/test/integration/embedding-resource-bounds.test.ts", - "status": "new", - "scenarios": [ - "Lazy 10k/100k/1m descriptors and oversized Unicode nodes: fixed queue high-water mark, identical text/chunks, no corpus materialization.", - "Smaller real-native case counts parent, graph/embedding child and sidecar peaks without double-counting thread RSS." - ] - }, - { - "file": "gitnexus/test/unit/vector-generation.test.ts", - "status": "existing", - "scenarios": [ - "Dirty/collapsed/mismatched graphs gate before native/model access; historical completed artifact is labeled preserved; pending job disables vector lane." - ] - }, - { - "file": "gitnexus/test/unit/local-backend-vector-generation.test.ts", - "status": "existing", - "scenarios": [ - "Preserved results hydrate current IDs and reject deleted nodes; active vector writer never opened; query ranking parity." - ] - }, - { - "file": "gitnexus/test/unit/embedding-job-progress.test.ts", - "status": "existing", - "scenarios": [ - "Completed+reused+pending+failed=total; active subset of pending; preserved is not current-complete; no false terminal event." - ] - }, - { - "file": "gitnexus/test/unit/server-sse-payload.test.ts", - "status": "existing", - "scenarios": [ - "Forward producer/IPC phase details through safe allowlist; graph-ready nonterminal; exactly one final success/failure/cancel event." - ] - }, - { - "file": "gitnexus/test/unit/analyze-memory-budget.test.ts", - "status": "existing", - "scenarios": [ - "Heap-only flag retains old meaning, including explicit ambient heap and respawn opt-out." - ] - }, - { - "file": "gitnexus/test/unit/analysis-resource-budget.test.ts", - "status": "existing", - "scenarios": [ - "Supervisor+children+native pools+sidecar+queues fit; defaults preserve useful concurrency when feasible." - ] - }, - { - "file": "gitnexus/test/unit/lbug-analysis-resources.test.ts", - "status": "existing", - "scenarios": [ - "Failed close remains charged; session drain blocks model/parser; cancellation during drain makes no metadata/provider calls." - ] - }, - { - "file": "gitnexus/test/integration/embedding-native-release.test.ts", - "status": "existing", - "scenarios": [ - "Explicit native pools are not reused before verified physical close; failed close retries, result-close failures clear active admission." - ] - }, - { - "file": "gitnexus/test/unit/embedding-sidecar-client.test.ts", - "status": "existing", - "scenarios": [ - "Parser phase waits for prior model-child exit; cancellation does not release a physically active request early." - ] - }, - { - "file": "gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts", - "status": "existing", - "scenarios": [ - "Keep legacy complete-group reuse, second-crash, changed/deleted nodes, force/drop behavior and known-identity relocation without provider requests." - ] - }, - { - "file": "gitnexus/test/integration/vector-generation-compatibility.test.ts", - "status": "new", - "scenarios": [ - "Plain analyze preserves legacy and previously completed vector coverage without automatic requests.", - "Unknown legacy provenance remains unknown; explicit generation logs any unsafe-reuse invalidation; relocation alone never invalidates verified groups.", - "Windows open handles, no VECTOR/FTS, shared sibling and external/branch storage, concurrent writers." - ] - }, - { - "file": "gitnexus/test/integration/shared-store-embedding-reuse.test.ts", - "status": "existing", - "scenarios": [ - "Prepared publication/pending vector work pins its artifacts; sibling cannot clean or mutate them." - ] - } - ], - "verification_commands": [ - "cd gitnexus && npm run build", - "cd gitnexus && npx tsc --noEmit", - "cd gitnexus && npm run typecheck:tests", - "cd gitnexus && npm test -- test/unit/analysis-publication.test.ts test/unit/vector-generation.test.ts test/unit/analyze-worker-core.test.ts test/unit/analyze-worker-ipc.test.ts test/unit/embedding-job-journal.test.ts test/unit/analysis-resource-budget.test.ts test/unit/lbug-analysis-resources.test.ts test/unit/server-sse-payload.test.ts", - "cd gitnexus && npm run test:integration -- test/integration/embedding-job-controller.test.ts test/integration/embedding-vector-store.test.ts test/integration/embedding-native-release.test.ts test/integration/analyze-staged-embedding-recovery.test.ts test/integration/shared-store-embedding-reuse.test.ts", - "cd gitnexus && npm test", - "cd gitnexus && node --import tsx bench/embedding-inputs/measure.cjs --check", - "cd gitnexus && node --import tsx bench/embedding-identities/measure.cjs --check" - ], - "risks": [ - "CRITICAL decideEmbeddingResume across analyze/sync/API; HIGH semanticSearch; previously CRITICAL native close/session helpers. Re-run impact before each edit.", - "Process handoff, publication, historical-coverage semantics, migration and rebase durability must activate coherently.", - "New process startup, manifest/native reopen, HTTP response parsing or pool sizing may regress latency/throughput; performance gate is unresolved." - ], - "assumptions": [ - "Whole-command performance is unmeasured. Verify with at least seven alternating pairs at equal work/resources before activation/shipment; no allowed slowdown threshold.", - "Automatic resource defaults are provisional. Include supervisor RSS, native pools, child/model runtime, queues and headroom; tune using the full matrix.", - "Windows and a representative local model have not been validated in this environment. Obtain those executions or report the acceptance gap; never infer a pass from Linux mocks.", - "PDG exists but current close slice is unavailable and controller output has mismatched source offsets plus truncation. Treat statement-level coverage as unresolved; source constraints remain authoritative.", - "Legacy model/text provenance may be absent. Preserve legacy read/preserve-only behavior; never invent a verified identity. Explicit new generation may invalidate only groups whose required compatibility cannot be proven, with reason and cost visible." - ], - "open_questions": [ - "No user product preference is required beyond the revised architecture gate. Performance defaults are decided by measurements, not an arbitrary regression allowance.", - "Windows/local-model evidence and PDG coverage limits remain explicit acceptance gaps." - ], - "avoid": [ - "Do not redo landed helper commits or repeat full repository discovery.", - "Do not force physical graph immutability/copying for default private incremental updates.", - "Do not invoke real native graph close on a CLI/server exit-only path just to begin embeddings; wait for process exit.", - "Do not serialize KnowledgeGraph/pipelineResult through IPC or add unbounded job-history scans.", - "Do not create a borrowed-index-lock token protocol; reacquire and revalidate the existing exclusive lock at handoff.", - "Do not mutate published vector artifacts, expose an active writer to readers, or present preserved historical coverage as current complete.", - "Do not charge provider work on plain preserve-only analyze, fabricate legacy identities, truncate input, or accept isolated chunk hashes as complete groups.", - "Do not change --memory-budget semantics or invalidate paid work because of scheduling/resource changes.", - "Do not terminate a native-active thread/worker; use cooperative safe points and verified exit.", - "Do not use microbenchmarks as proof of end-to-end non-regression or compare unequal work/resources.", - "Do not edit unrelated /workspace changes, dependencies, language-resolution behavior, CI/release/security config; no push or PR." - ], - "completed_commits": [ - { - "commit": "f0f3b9abe", - "result": "Original approved plan; superseded only by this Deepen revision." - }, - { - "commit": "60ddccefa", - "result": "Native storage feasibility and baseline microbenchmarks; whole-command harness still missing." - }, - { - "commit": "dbf62eab1", - "result": "Pure resource resolver and deterministic bounded manifest." - }, - { - "commit": "ec9063f65", - "result": "Vector store and durable segmented journal." - }, - { - "commit": "ba23148ba", - "result": "Bounded runner/text streaming and complete-group reuse." - }, - { - "commit": "084c52895", - "result": "Staged publication and generation-aware readers; in-place mode still needed." - }, - { - "commit": "f2fc492cf", - "result": "Published-job controller; no production callers yet." - }, - { - "commit": "8bdd572fd", - "result": "Optional parser/native resource allocation plumbing." - }, - { - "commit": "8dec4f273", - "result": "Physical sidecar reservations; parser admission not wired." - }, - { - "commit": "700d6aca8", - "result": "Readiness/progress consumers; producer/IPC wiring incomplete." - }, - { - "commit": "2363a1060", - "result": "Interrupted native bootstrap and journal recovery." - }, - { - "commit": "f953c569e", - "result": "Verified explicit native pool release and retry accounting." - }, - { - "commit": "8c6b680f3", - "result": "Main graph/session drain before in-process phase admission." - } - ], - "remaining_steps": [ - "Create full paired harness and re-anchor baseline", - "Extend graph publication and bounded exit handoff", - "Rebase pending artifacts and preserve legacy/historical coverage", - "Wire resource/progress/shared-store surfaces", - "Activate all three production writers together", - "Full regression/performance matrix and docs", - "GitNexus review; no push/PR" - ], - "index_refresh": { - "runner": "node gitnexus/dist/cli/index.js", - "storage": "isolated local navigation index", - "head": "8c6b680f3071025189804ffebac2fd3bc8bf1232", - "schema": 4, - "status": "up-to-date", - "runner_identity_status": "current", - "content_drift": "current", - "incomplete_reasons": [], - "receipt_match": "Exact MCP context runnerIdentity equals local status.index.runnerIdentity; local build/source current.", - "deepen_refreshes": 0, - "limitations": [ - "Connected MCP native engine uses storage42, local index storage47; use build-current local backend for graph calls.", - "Clusters/processes resource attempts failed engine mismatch; bounded local impact process summaries used instead.", - "PDG source offsets/truncation remain unresolved; no unrelated indexer repair included." - ] - }, - "validation_status": { - "feature": "incomplete; bounded controller not yet activated in production", - "latest_focused": "80 passed across six relevant suites for 8c6b680; build, source/test typecheck and lint passed (pre-existing warnings).", - "prior_focused": "f953c569: 265 passed, 1 skipped across recorded focused sets; 2363a106 bootstrap/journal regressions passed.", - "candidate_full_suite": "not run", - "performance": "microbenchmarks only; no whole-command non-regression acceptance", - "review": "gitnexus-review not run", - "platforms": "Linux native exercised; Windows/local-model unverified" - }, - "reanchor": { - "old_head": "f641342d1fcbfdcb764c5e478db0f69e6cea899a", - "new_head": "8c6b680f3071025189804ffebac2fd3bc8bf1232", - "old_plan_digest": "sha256:c36d6ad14288ca3ff0510c7d18a769d750c72475b5fff0fb476b10709e22d0e9", - "changed_cited_paths": [ - "gitnexus/src/cli/status.ts", - "gitnexus/src/core/analysis-publication.ts", - "gitnexus/src/core/analysis-resource-budget.ts", - "gitnexus/src/core/embeddings/embedding-job.ts", - "gitnexus/src/core/embeddings/embedding-pipeline.ts", - "gitnexus/src/core/embeddings/embedding-work-manifest.ts", - "gitnexus/src/core/embeddings/types.ts", - "gitnexus/src/core/index-freshness.ts", - "gitnexus/src/core/ingestion/workers/worker-pool.ts", - "gitnexus/src/core/lbug/lbug-adapter.ts", - "gitnexus/src/core/lbug/lbug-config.ts", - "gitnexus/src/core/lbug/pool-adapter.ts", - "gitnexus/src/core/run-analyze.ts", - "gitnexus/src/mcp/local/local-backend.ts", - "gitnexus/src/server/analyze-job.ts", - "gitnexus/src/server/api.ts", - "gitnexus/src/server/ops-snapshot.ts", - "gitnexus/src/storage/embedding-job.ts", - "gitnexus/src/storage/repo-manager.ts", - "gitnexus/src/storage/repo-meta.ts", - "gitnexus/src/storage/shared-store.ts", - "gitnexus/test/unit/analysis-resource-budget.test.ts", - "gitnexus/test/unit/embedding-work-manifest.test.ts", - "gitnexus/test/unit/server-sse-payload.test.ts" - ], - "policy": "Changed old ranges replaced by current seam evidence or retired; landed helpers reconciled rather than blindly re-pinned." - } - } -} -``` - -## 12. Assumptions and Open Questions - -- [assumed] Whole-command performance is unmeasured. Verify with at least seven alternating pairs at equal work/resources before activation/shipment; no allowed slowdown threshold. -- [assumed] Automatic resource defaults are provisional. Include supervisor RSS, native pools, child/model runtime, queues and headroom; tune using the full matrix. -- [assumed] Windows and a representative local model have not been validated in this environment. Obtain those executions or report the acceptance gap; never infer a pass from Linux mocks. -- [assumed] PDG exists but current close slice is unavailable and controller output has mismatched source offsets plus truncation. Treat statement-level coverage as unresolved; source constraints remain authoritative. -- [assumed] Legacy model/text provenance may be absent. Preserve legacy read/preserve-only behavior; never invent a verified identity. Explicit new generation may invalidate only groups whose required compatibility cannot be proven, with reason and cost visible. - -No new product preference is needed beyond approval of the concrete architecture revision. Implementation choices above are fixed; allocation tuning follows evidence. The full matrix, Windows/local-model coverage and final review remain completion gates, not reasons to claim a partial feature finished. - -The original stale-analyzer/graph-UID limitation has been replaced by a verified current local runner/index and explicit narrower PDG/resource limitations. Changed cited paths were reconciled before moving the evidence pin. Existing shared language-neutral ingestion rules and no-native-termination guardrails remain binding. - -Deferred: distributed embedding workers, new database engines, unrelated PDG fixes, graph-ingestion out-of-core redesign, automatic background scheduling and serving vectors from incomplete active jobs. Preserving a previously completed artifact with historical-coverage labeling is explicitly in scope for backward compatibility. - -The original `/workspace` user edits are outside this worktree/task. Do not stage them. No automatic push/PR. - -## 13. Definition of Done - -- [ ] New graph/keyword results are independently durable and readable before embeddings finish; every embedding failure preserves that graph. -- [ ] Staged and in-place crash semantics are tested and documented; tiny private incremental updates do not gain a whole-graph copy/hash. -- [ ] Exit-only graph workers physically exit before embedding admission; no native-active termination, unbounded IPC or double-counted resource reservations. -- [ ] All three production writers use bounded input/chunk/vector queues with identical text/coverage semantics; large-corpus and oversized-node cases pass. -- [ ] Compatible unchanged retries skip graph rebuilding and acknowledged provider requests; repeated source-change interruptions preserve all provably reusable paid groups. -- [ ] Plain analyze preserves existing vectors with zero automatic requests; current/preserved/legacy/pending coverage is truthful across CLI, MCP, HTTP and SSE. -- [ ] Force/drop, legacy recovery, shared/external/branch storage and Windows handle behavior pass their required cases; unmeasured platforms remain explicitly unresolved. -- [ ] Paired full-command evidence demonstrates no repeatable throughput/query-latency regression at comparable resources, including all added work and migration costs. -- [ ] Candidate full checks and final GitNexus review are complete; remaining findings are reported. No unrelated edits or unrequested publication. From 866b71b05a26664bb6321355950933429ae689b2 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 5 Nov 2026 20:10:42 +0000 Subject: [PATCH 42/67] ci: gate resumable analyze performance --- .github/workflows/ci-tests.yml | 69 +++ gitnexus/bench/analyze-resume/README.md | 129 ++--- gitnexus/bench/analyze-resume/baseline.json | 256 ++++++++++ gitnexus/bench/analyze-resume/measure.mjs | 57 ++- .../bench/analyze-resume/measure.test.mjs | 127 ----- .../bench/analyze-resume/native-baseline.json | 121 ----- .../analyze-resume/native-primitives.mjs | 220 --------- .../text-streaming-baseline.json | 440 ------------------ .../bench/analyze-resume/text-streaming.mjs | 102 ---- .../analyze-resume/vector-store-baseline.json | 60 --- .../bench/analyze-resume/vector-store.mjs | 168 ------- 11 files changed, 416 insertions(+), 1333 deletions(-) create mode 100644 gitnexus/bench/analyze-resume/baseline.json delete mode 100644 gitnexus/bench/analyze-resume/measure.test.mjs delete mode 100644 gitnexus/bench/analyze-resume/native-baseline.json delete mode 100644 gitnexus/bench/analyze-resume/native-primitives.mjs delete mode 100644 gitnexus/bench/analyze-resume/text-streaming-baseline.json delete mode 100644 gitnexus/bench/analyze-resume/text-streaming.mjs delete mode 100644 gitnexus/bench/analyze-resume/vector-store-baseline.json delete mode 100644 gitnexus/bench/analyze-resume/vector-store.mjs diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c0aa0fe3d..5373d9a74 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -7,6 +7,75 @@ permissions: contents: read jobs: + analyze-resume-benchmark: + name: analyze / resumable performance (${{ matrix.resources }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + resources: [explicit, default] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: Pin comparison revision + id: baseline + shell: bash + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [[ -n "$PR_BASE_SHA" ]]; then + comparison_sha="$PR_BASE_SHA" + else + comparison_sha="$(git rev-parse --verify HEAD^1)" + fi + printf 'sha=%s\n' "$comparison_sha" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/setup-gitnexus + with: + build: 'true' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.baseline.outputs.sha }} + path: baseline + persist-credentials: false + - name: Build baseline runtime + working-directory: baseline/gitnexus + run: | + npm ci + npm run build + npx tsx scripts/ensure-fts.ts + - name: Install candidate search extensions + working-directory: gitnexus + run: npx tsx scripts/ensure-fts.ts + - name: Check paired analyze and resume performance + working-directory: gitnexus + shell: bash + env: + RESOURCE_PROFILE: ${{ matrix.resources }} + run: | + mkdir -p "$RUNNER_TEMP/analyze-resume" + resource_args=() + if [[ "$RESOURCE_PROFILE" == default ]]; then + resource_args+=(--default-resources) + fi + node bench/analyze-resume/measure.mjs --check \ + --baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \ + --output "$RUNNER_TEMP/analyze-resume/results.json" \ + "${resource_args[@]}" 2>&1 | tee "$RUNNER_TEMP/analyze-resume/run.log" + - name: Retain paired measurements and analyzer logs + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: analyze-resume-performance-${{ matrix.resources }} + path: | + ${{ runner.temp }}/analyze-resume/*.json + ${{ runner.temp }}/analyze-resume/*.log + ${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.json + ${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.log + retention-days: 30 + lexical-lookup-benchmark: name: indexing / lexical lookup and cache interning runs-on: ubuntu-latest diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 90f10a3f2..8b5f1c052 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -1,104 +1,47 @@ -# Resumable analyze measurements +# Resumable analyze benchmark -Run the native feasibility check from `gitnexus/`: +Build two checkouts with matching dependencies on the same filesystem, then run +from the candidate's `gitnexus/` directory: ```sh -node --expose-gc bench/analyze-resume/native-primitives.mjs --check +node bench/analyze-resume/measure.mjs --check \ + --baseline /path/to/base/gitnexus/dist/cli/index.js \ + --output /tmp/analyze-resume-explicit.json ``` -The probe alternates baseline and candidate order, checks identical content and -vector coverage, verifies primary-key lookup in the native query plan, and opens -the published graph in a separate process while a vector writer remains open. -The write candidate includes active-intent fsync, a transaction per unit, native -checkpointing, and completed-cursor fsync. `--check` rejects a slower candidate -median for either reader or writer. +Repeat with `--default-resources` and a new output path to compare automatic +resource allocation. The explicit profile uses a 2 GiB heap, two parser workers, +and a 256 MiB native pool on both sides. Do not run indexing or tests alongside +these measurements. -`native-baseline.json` records the initial Linux/Node 22 samples. The bounded -reader retained at most 1 MiB of content per batch for 10,000 nodes containing -40.96 MB of content. It still holds a small key array to isolate native fetch -cost; production manifest memory must be tested separately. The writer probe -uses 512 vectors with 256 dimensions and 64 vectors per durability unit. +`baseline.json` records the corpus settings, seven measured pairs per scenario, +commit IDs, and original Linux/Node 22 results. Its historical timings are +provenance, not portable wall-clock limits. The runner compares the supplied base +and candidate in alternating order, after a warmup, on the current machine. -These measurements establish native feasibility. They do not establish -end-to-end analyze throughput, crash recovery, model throughput, or total RSS -bounds. Run the application regression matrix before making those claims. -Windows handle behavior and local-model execution also need their own runs. +The six CI scenarios cover graph-only analysis, warm no-op, an incremental edit, +fresh HTTP embeddings, cached HTTP embeddings, and retry after a deterministic +provider outage. Setup runs are outside the measured interval. Assertions require +equal graph/vector counts and embedding-input fingerprints, matching runtime +dependencies, completed publication, and no paid requests for a cached run. -Measure complete CLI invocations with two built checkouts: +`--check` requires all six scenarios, the recorded corpus size, a warmup and at +least seven pairs. It fails when the paired 95% bootstrap interval lies entirely +above the candidate/base ratio of **1.0**. An interval crossing 1 remains +**inconclusive** and is reported as such; a successful CI gate means no slowdown +was statistically detected, not proof of equal performance. The original +graph-only interval in `baseline.json` is inconclusive and has not been discarded. -```sh -node bench/analyze-resume/measure.mjs \ - --baseline /path/to/baseline/gitnexus/dist/cli/index.js \ - --output /tmp/analyze-pairs.json -``` +`.github/workflows/ci-tests.yml` runs both resource profiles against the PR base +(the first parent on non-PR runs). Failures block the existing CI Gate. Each job +uploads raw samples, summaries, dependency receipts and analyzer logs, including +partial results on failure. The runner never overwrites an existing report. -The harness creates isolated Git fixtures, registries and stores, alternates -baseline/candidate order, warms each scenario and records seven measured pairs. -Stage both built runtimes on the same filesystem with matching dependencies. -Preflight rejects different runtime devices/filesystem types, dependency -lockfiles or native entry bytes; each completed pair also compares the full -resolved dependency digest from its analyzer receipts. A checkout on a remote -filesystem compared with a local checkout is not useful regression evidence. -It includes CLI startup, graph writes, embeddings and final index work. Scenarios -cover graph-only, warm no-op, one-function incremental edits, fresh/cached HTTP -embeddings, local-model fresh/cached runs, and resume after a deterministic provider outage. The outage ends -normally through application error handling; it does not kill native work. -`--help` lists corpus, heap, worker and scenario options. Both sides receive the -same explicit allocations. Run a separate comparison with `--default-resources` -to omit heap, worker and native-pool overrides on both sides. Local scenarios -use the installed default model; download it before timing to keep network -transfer out of the comparison. +Benchmark execution belongs exclusively to the dedicated CI jobs above. It is +not included in the unit or integration test suites. -Raw samples include command/log paths, analyzer receipts, graph/vector counts, -provider request/input counts, deterministic input fingerprints and sampled Linux -process-family RSS. Fixture setup is outside the timer and recorded separately. -Logs and measurements remain beside the output; native stores are removed after -each measurement unless `--keep-stores` is set. An existing output is never -overwritten. No-op/cached runs and equal-work checks fail on unexpected work. -Sending SIGINT to the harness PID stops between invocations after the active CLI -exits naturally; the harness does not forward a termination signal to native work. - -Results use paired log-ratio bootstrap intervals and report regression, -inconclusive, or no detected regression. Fewer than seven pairs is always -smoke-only. The report always marks overall acceptance incomplete: synthetic -counts/input parity do not establish query ranking, -concurrent query latency, exact peak RSS, queue/native reservations, CPU/disk costs, -unselected scenarios/resource profiles or Windows correctness. Those remain part of the final -regression/performance matrix. Do not run measurements alongside indexing/tests. - -Check the harness itself with: - -```sh -node --test bench/analyze-resume/measure.test.mjs -``` - -Measure the implemented vector store and journal after building: - -```sh -npm run build -node bench/analyze-resume/vector-store.mjs --check -``` - -This alternates five baseline and candidate runs over 512 nodes and 256-dimensional -vectors. The baseline uses the existing prepared DELETE/CREATE write sequence. -The candidate includes node receipts, complete chunk-set validation, a native -checkpoint per 64-node unit, and fsynced intent/acknowledgement journal updates. -Schema creation and final index construction are outside the timed region for -both paths. This isolates write overhead; it does not measure inference or graph -ingestion. `--check` rejects a candidate median above the baseline median. -The initial Linux/Node 22 run in `vector-store-baseline.json` measured a 0.470 -candidate/baseline median ratio, including the durability journal. - -Compare text preparation and streaming chunks with an unchanged built checkout: - -```sh -node --expose-gc bench/analyze-resume/text-streaming.mjs /path/to/baseline/gitnexus/dist -``` - -The probe warms both implementations, alternates seven measured pairs, and checks -identical content hashes and every chunk's text, offsets, and line spans. The -baseline checkout for `text-streaming-baseline.json` was commit -`f641342d1fcbfdcb764c5e478db0f69e6cea899a`. Candidate/baseline median ratios were -0.957 for ordinary snippets, 0.948 for many short lines, 0.917 for an oversized -short-label node, 0.919 for a large declaration, and 0.595 for character chunks. -These are preparation costs, not end-to-end inference or analysis measurements. +`--help` lists additional options, including optional local-model scenarios. +Synthetic fixtures do not establish query ranking, concurrent query latency, +local-model throughput, exact peak RSS or platform correctness. Reports keep +overall acceptance marked incomplete. SIGINT sent to the harness stops between +invocations and lets any active native CLI exit naturally. diff --git a/gitnexus/bench/analyze-resume/baseline.json b/gitnexus/bench/analyze-resume/baseline.json new file mode 100644 index 000000000..82f6ac52e --- /dev/null +++ b/gitnexus/bench/analyze-resume/baseline.json @@ -0,0 +1,256 @@ +{ + "schemaVersion": 1, + "maxPairedRatio": 1, + "settings": { + "pairs": 7, + "warmups": 1, + "modules": 64, + "functions": 16, + "heap-mb": 2048, + "workers": 2, + "sample-ms": 25 + }, + "reference": { + "measuredAt": "2026-11-05T19:17:51.910Z", + "baselineCommit": "a0586d967f391f6f78e24d8c7162abfe350ae195", + "candidateCommit": "8c1b489c962a223b77567c21dcd1a9f2c9788851", + "environment": { + "node": "v22.23.3", + "platform": "linux", + "architecture": "x64", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "resourceProfile": "explicit-heap-workers-pool" + }, + "note": "Historical samples are provenance, not portable wall-clock limits. CI compares the current PR base and candidate on the same runner. An interval crossing 1 remains inconclusive." + }, + "scenarios": { + "graph-only": { + "pairs": 7, + "baselineMedianMs": 11254.142376000003, + "candidateMedianMs": 10191.042332999998, + "geometricRatio": 1.0582904590478737, + "pairedBootstrap95": [0.9484949043112423, 1.2159205271380682], + "evidence": "inconclusive", + "samplesMs": [ + { + "baseline": 11254.142376000003, + "candidate": 10144.616939999996 + }, + { + "baseline": 10671.354456000001, + "candidate": 9944.314222 + }, + { + "baseline": 10188.076402000006, + "candidate": 10191.042332999998 + }, + { + "baseline": 11853.487859999994, + "candidate": 17919.62693900001 + }, + { + "baseline": 11739.811645999987, + "candidate": 14308.022194000005 + }, + { + "baseline": 10668.565815000009, + "candidate": 10129.661259000015 + }, + { + "baseline": 11425.665953999996, + "candidate": 11556.346957000002 + } + ] + }, + "noop": { + "pairs": 7, + "baselineMedianMs": 2184.29468500003, + "candidateMedianMs": 1950.1828749999986, + "geometricRatio": 0.8930352942528001, + "pairedBootstrap95": [0.8795970798771887, 0.905936077701467], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 2198.722995999997, + "candidate": 1915.772708000004 + }, + { + "baseline": 2239.4145290000015, + "candidate": 1939.7219499999774 + }, + { + "baseline": 2176.4498259999673, + "candidate": 1950.1828749999986 + }, + { + "baseline": 2184.29468500003, + "candidate": 2005.5672990000458 + }, + { + "baseline": 2179.4396279999637, + "candidate": 1966.6888930000132 + }, + { + "baseline": 2128.4910190000082, + "candidate": 1891.617927000043 + }, + { + "baseline": 2208.1911739999778, + "candidate": 2008.7850609999732 + } + ] + }, + "incremental": { + "pairs": 7, + "baselineMedianMs": 5783.258446000051, + "candidateMedianMs": 5541.312566999986, + "geometricRatio": 0.9674187960203702, + "pairedBootstrap95": [0.9561405967756264, 0.9816590666894027], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 5779.961653999984, + "candidate": 5672.744095000031 + }, + { + "baseline": 5806.2170550000155, + "candidate": 5541.312566999986 + }, + { + "baseline": 5861.75522000005, + "candidate": 5892.487050000054 + }, + { + "baseline": 5720.598826000001, + "candidate": 5503.345939000021 + }, + { + "baseline": 5744.461226000101, + "candidate": 5528.829626999912 + }, + { + "baseline": 5783.258446000051, + "candidate": 5528.050583000062 + }, + { + "baseline": 5888.105557000032, + "candidate": 5603.379379000049 + } + ] + }, + "http-fresh": { + "pairs": 7, + "baselineMedianMs": 15511.024734000093, + "candidateMedianMs": 14335.91775100003, + "geometricRatio": 0.9272010938503702, + "pairedBootstrap95": [0.918360962384628, 0.9365473751189902], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 15511.024734000093, + "candidate": 14335.91775100003 + }, + { + "baseline": 15510.167916000006, + "candidate": 14261.329035999952 + }, + { + "baseline": 15609.408870000043, + "candidate": 14153.163587999996 + }, + { + "baseline": 15409.513349999906, + "candidate": 14634.943147000042 + }, + { + "baseline": 15519.041560999933, + "candidate": 14481.790279999957 + }, + { + "baseline": 15767.326278000022, + "candidate": 14690.776563999942 + }, + { + "baseline": 15408.597242999938, + "candidate": 14267.204865999985 + } + ] + }, + "http-cached": { + "pairs": 7, + "baselineMedianMs": 2149.4189200000837, + "candidateMedianMs": 1919.7719739999156, + "geometricRatio": 0.8972216499596883, + "pairedBootstrap95": [0.8876158802568739, 0.9067963765721975], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 2165.3657219999004, + "candidate": 1909.4434619999956 + }, + { + "baseline": 2132.151501000044, + "candidate": 1941.053768999991 + }, + { + "baseline": 2141.9542270000093, + "candidate": 1934.437436999986 + }, + { + "baseline": 2155.985989000066, + "candidate": 1919.7719739999156 + }, + { + "baseline": 2113.9462319999, + "candidate": 1902.646522999974 + }, + { + "baseline": 2149.4189200000837, + "candidate": 1968.86528300005 + }, + { + "baseline": 2152.111167999916, + "candidate": 1892.6074149999768 + } + ] + }, + "http-resume": { + "pairs": 7, + "baselineMedianMs": 15869.910479000071, + "candidateMedianMs": 4004.7973079998046, + "geometricRatio": 0.2485479851486286, + "pairedBootstrap95": [0.24355429100174475, 0.25278581390854993], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 15779.565613000188, + "candidate": 4038.0054809998255 + }, + { + "baseline": 15869.910479000071, + "candidate": 4017.237259999849 + }, + { + "baseline": 15861.843901999993, + "candidate": 3971.7236019999254 + }, + { + "baseline": 15861.11537200003, + "candidate": 3973.9850310001057 + }, + { + "baseline": 17113.716924000066, + "candidate": 4044.0885320000816 + }, + { + "baseline": 16508.58412100002, + "candidate": 4004.7973079998046 + }, + { + "baseline": 15901.872792999959, + "candidate": 3999.7262199998368 + } + ] + } + } +} diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs index 6f458a628..74728421c 100644 --- a/gitnexus/bench/analyze-resume/measure.mjs +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -124,6 +124,31 @@ export function summarizePairs(pairs) { }; } +/** Detect slowdowns against a contemporaneous base, never historical wall-clock timings. */ +export function checkPerformance(summaries, baseline) { + assert.equal(baseline.schemaVersion, 1, 'Unsupported benchmark baseline'); + assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression'); + const scenarios = Object.keys(baseline.scenarios); + assert(scenarios.length > 0, 'Missing baseline scenarios'); + const failedScenarios = []; + const inconclusiveScenarios = []; + for (const scenario of scenarios) { + const summary = summaries[scenario]; + assert(summary && summary.pairs >= 7, `Missing seven measured pairs for ${scenario}`); + const interval = summary.pairedBootstrap95; + assert( + Array.isArray(interval) && + interval.length === 2 && + interval.every((value) => Number.isFinite(value) && value > 0) && + interval[0] <= interval[1], + `Invalid paired interval for ${scenario}`, + ); + if (interval[0] > baseline.maxPairedRatio) failedScenarios.push(scenario); + else if (interval[1] >= baseline.maxPairedRatio) inconclusiveScenarios.push(scenario); + } + return { failedScenarios, inconclusiveScenarios }; +} + /** Sample Linux process RSS once per process, including descendants, never per thread. */ async function sampleTree(pid) { if (process.platform !== 'linux') return undefined; @@ -443,13 +468,16 @@ async function runCase({ } export async function main(argv = process.argv.slice(2)) { + const baseline = JSON.parse( + await fs.readFile(new URL('./baseline.json', import.meta.url), 'utf8'), + ); const { values: v } = parseArgs({ args: argv, options: { baseline: { type: 'string' }, candidate: { type: 'string' }, output: { type: 'string' }, - scenarios: { type: 'string', default: SCENARIOS.join(',') }, + scenarios: { type: 'string', default: Object.keys(baseline.scenarios).join(',') }, pairs: { type: 'string', default: '7' }, warmups: { type: 'string', default: '1' }, modules: { type: 'string', default: '64' }, @@ -459,6 +487,7 @@ export async function main(argv = process.argv.slice(2)) { 'default-resources': { type: 'boolean', default: false }, 'sample-ms': { type: 'string', default: '25' }, 'keep-stores': { type: 'boolean', default: false }, + check: { type: 'boolean', default: false }, help: { type: 'boolean' }, }, }); @@ -469,6 +498,7 @@ export async function main(argv = process.argv.slice(2)) { --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 --default-resources Omit heap, worker and native-pool overrides on both sides + --check Require the baseline corpus/scenarios and reject detected slowdowns --keep-stores Retain owned index/home artifacts (logs and measurements always retained) Includes full CLI startup, graph/manifest/native writes and final index work. HTTP resume uses a deterministic provider outage, not an unsafe native-process kill. @@ -502,6 +532,16 @@ Live-query latency and platform cases remain separate acceptance requirements.`) scenarios.every((x) => SCENARIOS.includes(x)), 'Invalid or repeated scenario', ); + if (v.check) { + assert(numbers.pairs >= 7 && numbers.warmups >= 1, '--check requires seven pairs and a warmup'); + for (const key of ['modules', 'functions']) + assert.equal(numbers[key], baseline.settings[key], `--check requires baseline ${key}`); + assert( + Object.keys(baseline.scenarios).every((scenario) => scenarios.includes(scenario)), + '--check requires all baseline scenarios', + ); + assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression'); + } const candidate = v.candidate ?? fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url)); const clis = { baseline: path.resolve(v.baseline), candidate: path.resolve(candidate) }; @@ -634,9 +674,22 @@ Live-query latency and platform cases remain separate acceptance requirements.`) report.summaries[scenario] = summarizePairs(pairs); await save(); } + if (v.check) { + report.check = checkPerformance(report.summaries, baseline); + await save(); + assert.equal( + report.check.failedScenarios.length, + 0, + `Performance regression: ${report.check.failedScenarios.join(', ')}; inspect ${output}`, + ); + if (report.check.inconclusiveScenarios.length) + console.warn( + `Inconclusive timings: ${report.check.inconclusiveScenarios.join(', ')}; not proof of equal performance.`, + ); + } console.log( JSON.stringify( - { output, summaries: report.summaries, acceptance: report.acceptance }, + { output, summaries: report.summaries, check: report.check, acceptance: report.acceptance }, null, 2, ), diff --git a/gitnexus/bench/analyze-resume/measure.test.mjs b/gitnexus/bench/analyze-resume/measure.test.mjs deleted file mode 100644 index 6eecfbb2a..000000000 --- a/gitnexus/bench/analyze-resume/measure.test.mjs +++ /dev/null @@ -1,127 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; -import { test } from 'node:test'; -import { - assertComparableRuntimes, - assertCompletedCase, - measureProcess, - startProvider, - summarizePairs, -} from './measure.mjs'; - -test('incomparable runtime filesystems or dependency bytes cannot produce paired evidence', () => { - const baseline = { - cli: { device: '1', filesystem: 'ext4' }, - nativeEntry: { device: '1', filesystem: 'ext4' }, - lockDigest: 'lock', - nativeEntryDigest: 'native', - }; - assertComparableRuntimes(baseline, structuredClone(baseline)); - const remote = structuredClone(baseline); - remote.cli.device = '2'; - remote.cli.filesystem = 'v9fs'; - assert.throws(() => assertComparableRuntimes(baseline, remote), /different devices/); - assert.throws( - () => assertComparableRuntimes(baseline, { ...baseline, lockDigest: 'other' }), - /lockfiles differ/, - ); - assert.throws( - () => assertComparableRuntimes(baseline, { ...baseline, nativeEntryDigest: 'other' }), - /entry bytes differ/, - ); -}); - -test('completion checks reject a rebuilding no-op and an incomplete successful embedding run', () => { - const metadata = { indexedAt: 'first', stats: { nodes: 10, embeddings: 5 } }; - assertCompletedCase('noop', metadata, metadata); - assert.throws( - () => assertCompletedCase('noop', { ...metadata, indexedAt: 'second' }, metadata), - /No-op changed/, - ); - assert.throws( - () => assertCompletedCase('http-fresh', { ...metadata, embeddingJob: {} }), - /pending/, - ); - assert.throws( - () => assertCompletedCase('http-resume', { ...metadata, embeddingCheckpoint: {} }), - /pending/, - ); - assert.throws( - () => assertCompletedCase('http-fresh', { stats: { nodes: 10, embeddings: 0 } }), - /no vectors/, - ); -}); - -test('paired evidence never treats small samples or equality as acceptance', () => { - assert.equal(summarizePairs([{ baseline: 100, candidate: 90 }]).evidence, 'smoke-only'); - const equal = Array.from({ length: 7 }, () => ({ baseline: 100, candidate: 100 })); - assert.equal(summarizePairs(equal).evidence, 'inconclusive'); - assert.equal(summarizePairs(equal.map((p) => ({ ...p, candidate: 110 }))).evidence, 'regression'); - assert.equal( - summarizePairs(equal.map((p) => ({ ...p, candidate: 90 }))).evidence, - 'no-detected-regression', - ); - assert.throws(() => summarizePairs([{ baseline: 0, candidate: 1 }])); - assert.throws(() => summarizePairs([])); -}); - -test('process measurement waits for completion, records failure and preserves logs', async () => { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'analyze-measure-test-')); - try { - const logFile = path.join(directory, 'child.log'); - const result = await measureProcess( - process.execPath, - ['-e', 'setTimeout(() => { console.log("finished"); process.exitCode = 7; }, 70)'], - { cwd: directory, env: process.env, logFile, sampleMs: 10 }, - ); - assert.equal(result.exitCode, 7); - assert.equal(result.signal, null); - assert(result.elapsedMs >= 70); - assert.equal((await fs.readFile(logFile, 'utf8')).trim(), 'finished'); - if (process.platform === 'linux') { - assert(result.rssSamples > 0); - assert(result.sampledPeakProcessTreeRssBytes > 0); - } - await assert.rejects( - () => - measureProcess(process.execPath, ['-e', ''], { - cwd: directory, - env: process.env, - logFile, - }), - { code: 'EEXIST' }, - ); - assert.equal((await fs.readFile(logFile, 'utf8')).trim(), 'finished'); - } finally { - await fs.rm(directory, { recursive: true, force: true }); - } -}); - -test('provider outage preserves paid-input accounting and deterministic vectors', async () => { - const provider = await startProvider(16); - const request = (input) => - fetch(`${provider.url}/embeddings`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ input }), - }); - try { - const counters = provider.begin(2); - const first = await (await request(['alpha', 'beta'])).json(); - const failure = await request(['gamma']); - assert.equal(failure.status, 503); - await failure.json(); - assert.equal(counters.acceptedInputs, 2); - assert.equal(counters.rejectedRequests, 1); - counters.failAfter = Infinity; - const retry = await (await request(['alpha'])).json(); - assert.deepEqual(retry.data[0].embedding, first.data[0].embedding); - assert.equal(counters.acceptedInputs, 3); - assert.equal(counters.inputs.size, 2); - assert.equal(counters.requests, 3); - } finally { - await provider.close(); - } -}); diff --git a/gitnexus/bench/analyze-resume/native-baseline.json b/gitnexus/bench/analyze-resume/native-baseline.json deleted file mode 100644 index 8e4a439ee..000000000 --- a/gitnexus/bench/analyze-resume/native-baseline.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "node": "v22.23.3", - "platform": "linux", - "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", - "nodes": 10000, - "unit": 256, - "vectorNodes": 512, - "dimensions": 256, - "samples": [ - { - "phase": "read", - "trial": 0, - "mode": "baseline", - "ms": 560.4716119999999, - "maximumContentBytes": 40960000 - }, - { - "phase": "read", - "trial": 0, - "mode": "bounded", - "ms": 84.59907299999998, - "maximumContentBytes": 1048576 - }, - { - "phase": "read", - "trial": 1, - "mode": "bounded", - "ms": 75.56121000000007, - "maximumContentBytes": 1048576 - }, - { - "phase": "read", - "trial": 1, - "mode": "baseline", - "ms": 601.821259, - "maximumContentBytes": 40960000 - }, - { - "phase": "read", - "trial": 2, - "mode": "baseline", - "ms": 597.1018180000001, - "maximumContentBytes": 40960000 - }, - { - "phase": "read", - "trial": 2, - "mode": "bounded", - "ms": 76.60035700000026, - "maximumContentBytes": 1048576 - }, - { - "phase": "read", - "trial": 3, - "mode": "bounded", - "ms": 77.93451700000014, - "maximumContentBytes": 1048576 - }, - { - "phase": "read", - "trial": 3, - "mode": "baseline", - "ms": 605.307159, - "maximumContentBytes": 40960000 - }, - { - "phase": "read", - "trial": 4, - "mode": "baseline", - "ms": 566.7196509999999, - "maximumContentBytes": 40960000 - }, - { - "phase": "read", - "trial": 4, - "mode": "bounded", - "ms": 68.64343800000006, - "maximumContentBytes": 1048576 - }, - { - "phase": "write", - "trial": 0, - "mode": "baseline", - "ms": 3262.1321529999996 - }, - { - "phase": "write", - "trial": 0, - "mode": "bounded", - "ms": 683.4758389999997 - }, - { - "phase": "write", - "trial": 1, - "mode": "bounded", - "ms": 692.3224590000009 - }, - { - "phase": "write", - "trial": 1, - "mode": "baseline", - "ms": 3120.299986 - }, - { - "phase": "write", - "trial": 2, - "mode": "baseline", - "ms": 2709.020754000001 - }, - { - "phase": "write", - "trial": 2, - "mode": "bounded", - "ms": 642.1225830000003 - } - ], - "ratios": { - "read": 0.12828692643504938, - "write": 0.21904170819042518 - } -} diff --git a/gitnexus/bench/analyze-resume/native-primitives.mjs b/gitnexus/bench/analyze-resume/native-primitives.mjs deleted file mode 100644 index 1ac5512ba..000000000 --- a/gitnexus/bench/analyze-resume/native-primitives.mjs +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env node -// Native feasibility probe, not an end-to-end analyze performance claim. -// node --expose-gc bench/analyze-resume/native-primitives.mjs [--check] -import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; -import { createRequire } from 'node:module'; -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; - -const require = createRequire(import.meta.url); -const { Database, Connection } = require('@ladybugdb/core'); -const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-analyze-resume-')); -const nodes = 10_000; -const content = 'x'.repeat(4096); -const unit = 256; -const vectorNodes = 512; -const dimensions = 256; -const samples = []; - -async function query(connection, text, parameters) { - const result = parameters - ? await connection.execute(await connection.prepare(text), parameters) - : await connection.query(text); - try { - return await result.getAll(); - } finally { - result.close(); - } -} - -async function durableJson(file, value) { - const handle = await fs.open(`${file}.tmp`, 'w'); - try { - await handle.writeFile(JSON.stringify(value)); - await handle.sync(); - } finally { - await handle.close(); - } - await fs.rename(`${file}.tmp`, file); - // Directory fsync is unavailable on Windows; file fsync still runs there. - if (process.platform !== 'win32') { - const directory = await fs.open(path.dirname(file), 'r'); - try { - await directory.sync(); - } finally { - await directory.close(); - } - } -} - -const graphPath = path.join(root, 'graph'); -let graph; -let graphConnection; -try { - graph = new Database(graphPath, 128 * 1024 * 1024); - await graph.init(); - graphConnection = new Connection(graph, 2); - await query(graphConnection, 'CREATE NODE TABLE N(id STRING, content STRING, PRIMARY KEY(id))'); - await query( - graphConnection, - `UNWIND range(0, ${nodes - 1}) AS i CREATE (:N {id: cast(i AS STRING), content: $content})`, - { content }, - ); - await query(graphConnection, 'CHECKPOINT'); - await graphConnection.close(); - await graph.close(); - graph = new Database(graphPath, 128 * 1024 * 1024, true, true); - await graph.init(); - graphConnection = new Connection(graph, 2); - const fetchText = 'UNWIND $ids AS id MATCH (n:N {id:id}) RETURN n.id AS id, n.content AS content'; - const fetch = await graphConnection.prepare(fetchText); - const plan = await query(graphConnection, `EXPLAIN ${fetchText}`, { ids: ['0'] }); - assert.match(JSON.stringify(plan), /PRIMARY_KEY_LOOKUP/); - - for (let trial = 0; trial < 5; trial++) { - for (const mode of trial % 2 ? ['bounded', 'baseline'] : ['baseline', 'bounded']) { - global.gc?.(); - const start = performance.now(); - let bytes = 0; - let maximumContentBytes = 0; - if (mode === 'baseline') { - const rows = await query( - graphConnection, - 'MATCH (n:N) RETURN n.id AS id, n.content AS content', - ); - for (const row of rows) bytes += Buffer.byteLength(row.content); - maximumContentBytes = bytes; - } else { - // This probe isolates native fetch cost. Production must persist keys - // in its manifest; this small key array is not a bounded manifest. - const keys = []; - const result = await graphConnection.query('MATCH (n:N) RETURN n.id AS id ORDER BY n.id'); - try { - while (result.hasNext()) keys.push(result.getNextSync().id); - } finally { - result.close(); - } - for (let offset = 0; offset < keys.length; offset += unit) { - const rows = await graphConnection.execute(fetch, { - ids: keys.slice(offset, offset + unit), - }); - let batchBytes = 0; - try { - while (rows.hasNext()) batchBytes += Buffer.byteLength(rows.getNextSync().content); - } finally { - rows.close(); - } - bytes += batchBytes; - maximumContentBytes = Math.max(maximumContentBytes, batchBytes); - await new Promise((resolve) => setImmediate(resolve)); - } - assert.ok(maximumContentBytes <= unit * Buffer.byteLength(content)); - } - assert.equal(bytes, nodes * Buffer.byteLength(content)); - samples.push({ - phase: 'read', - trial, - mode, - ms: performance.now() - start, - maximumContentBytes, - }); - } - } - - for (let trial = 0; trial < 3; trial++) { - for (const mode of trial % 2 ? ['bounded', 'baseline'] : ['baseline', 'bounded']) { - const vectors = new Database(path.join(root, `vectors-${trial}-${mode}`), 64 * 1024 * 1024); - await vectors.init(); - const writer = new Connection(vectors, 2); - try { - await query( - writer, - `CREATE NODE TABLE E(id STRING, embedding FLOAT[${dimensions}], PRIMARY KEY(id))`, - ); - // A fresh process must be able to read the published graph while the - // vector writer is open. Same-process handle sharing cannot prove this. - const child = execFileSync( - process.execPath, - [ - '-e', - ` - const {Database,Connection}=require(${JSON.stringify(require.resolve('@ladybugdb/core'))}); - (async()=>{const db=new Database(${JSON.stringify(graphPath)},67108864,true,true); - await db.init();const c=new Connection(db,2); - const r=await c.query('MATCH (n:N) RETURN count(n) AS count'); - require('node:fs').writeSync(1,JSON.stringify(await r.getAll()));r.close(); - await c.close();await db.close();})().catch(e=>{console.error(e);process.exitCode=1}); - `, - ], - { encoding: 'utf8', timeout: 30_000 }, - ); - assert.equal(JSON.parse(child)[0].count, nodes); - const insert = await writer.prepare('CREATE (:E {id:$id, embedding:$embedding})'); - const embedding = Array(dimensions).fill(0.125); - const start = performance.now(); - for (let offset = 0; offset < vectorNodes; offset += 64) { - if (mode === 'bounded') { - await durableJson(path.join(root, 'job.json'), { - completed: offset, - active: offset / 64, - }); - await query(writer, 'BEGIN TRANSACTION'); - } - for (let i = offset; i < offset + 64; i++) { - const result = await writer.execute(insert, { id: String(i), embedding }); - result.close(); - } - if (mode === 'bounded') { - await query(writer, 'COMMIT'); - await query(writer, 'CHECKPOINT'); - await durableJson(path.join(root, 'job.json'), { completed: offset + 64 }); - } - } - await query(writer, 'CHECKPOINT'); - samples.push({ phase: 'write', trial, mode, ms: performance.now() - start }); - assert.equal( - (await query(writer, 'MATCH (n:E) RETURN count(n) AS count'))[0].count, - vectorNodes, - ); - } finally { - await writer.close(); - await vectors.close(); - } - } - } - const median = (values) => values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]; - const ratios = Object.fromEntries( - ['read', 'write'].map((phase) => [ - phase, - median(samples.filter((s) => s.phase === phase && s.mode === 'bounded').map((s) => s.ms)) / - median(samples.filter((s) => s.phase === phase && s.mode === 'baseline').map((s) => s.ms)), - ]), - ); - console.log( - JSON.stringify( - { - node: process.version, - platform: process.platform, - cpu: os.cpus()[0]?.model, - nodes, - unit, - vectorNodes, - dimensions, - samples, - ratios, - }, - null, - 2, - ), - ); - if (process.argv.includes('--check')) { - for (const [phase, ratio] of Object.entries(ratios)) - assert.ok(ratio <= 1, `${phase} regressed: ${ratio}`); - } -} finally { - await graphConnection?.close(); - await graph?.close(); - await fs.rm(root, { recursive: true, force: true }); -} diff --git a/gitnexus/bench/analyze-resume/text-streaming-baseline.json b/gitnexus/bench/analyze-resume/text-streaming-baseline.json deleted file mode 100644 index 020d4ce7f..000000000 --- a/gitnexus/bench/analyze-resume/text-streaming-baseline.json +++ /dev/null @@ -1,440 +0,0 @@ -{ - "node": "v22.23.3", - "platform": "linux", - "fingerprints": { - "short-code": "6f3ccc2ffd9833584b144a688e0d711911ba506c", - "many-short-lines": "b903948f55a0e163eeed5d3f1dae1a3ada34b011", - "large-short-label": "f10a16214a869c4aa62fb0afc926bae99db60fef", - "large-declaration": "ce8401d3b4df5cc25d8378323ff056ca101d04b2", - "chunks": "78e8ab79f72aeea64cd8b1aaa3aa50c766cb1a7cf6402e61a72d3f81cca496e8" - }, - "ratios": { - "short-code": 0.9570562752783565, - "many-short-lines": 0.9482353771383033, - "large-short-label": 0.9167695057681882, - "large-declaration": 0.9190759176146885, - "chunks": 0.5951500187087568 - }, - "samples": [ - { - "case": "short-code", - "trial": 0, - "mode": "baseline", - "ms": 15.460057000000234 - }, - { - "case": "short-code", - "trial": 0, - "mode": "candidate", - "ms": 14.650719000001118 - }, - { - "case": "short-code", - "trial": 1, - "mode": "candidate", - "ms": 16.895096999998714 - }, - { - "case": "short-code", - "trial": 1, - "mode": "baseline", - "ms": 16.1013669999993 - }, - { - "case": "short-code", - "trial": 2, - "mode": "baseline", - "ms": 15.971506999998383 - }, - { - "case": "short-code", - "trial": 2, - "mode": "candidate", - "ms": 15.285631000000649 - }, - { - "case": "short-code", - "trial": 3, - "mode": "candidate", - "ms": 16.652016000000003 - }, - { - "case": "short-code", - "trial": 3, - "mode": "baseline", - "ms": 15.748994000001403 - }, - { - "case": "short-code", - "trial": 4, - "mode": "baseline", - "ms": 19.216942000000927 - }, - { - "case": "short-code", - "trial": 4, - "mode": "candidate", - "ms": 15.05308899999909 - }, - { - "case": "short-code", - "trial": 5, - "mode": "candidate", - "ms": 14.682182999998986 - }, - { - "case": "short-code", - "trial": 5, - "mode": "baseline", - "ms": 18.089461999999912 - }, - { - "case": "short-code", - "trial": 6, - "mode": "baseline", - "ms": 14.437770000000455 - }, - { - "case": "short-code", - "trial": 6, - "mode": "candidate", - "ms": 15.846268000001146 - }, - { - "case": "many-short-lines", - "trial": 0, - "mode": "baseline", - "ms": 222.07076000000052 - }, - { - "case": "many-short-lines", - "trial": 0, - "mode": "candidate", - "ms": 204.5821530000012 - }, - { - "case": "many-short-lines", - "trial": 1, - "mode": "candidate", - "ms": 199.73077399999966 - }, - { - "case": "many-short-lines", - "trial": 1, - "mode": "baseline", - "ms": 203.6863659999999 - }, - { - "case": "many-short-lines", - "trial": 2, - "mode": "baseline", - "ms": 203.15121700000054 - }, - { - "case": "many-short-lines", - "trial": 2, - "mode": "candidate", - "ms": 194.02467500000057 - }, - { - "case": "many-short-lines", - "trial": 3, - "mode": "candidate", - "ms": 200.72532900000078 - }, - { - "case": "many-short-lines", - "trial": 3, - "mode": "baseline", - "ms": 208.93676400000004 - }, - { - "case": "many-short-lines", - "trial": 4, - "mode": "baseline", - "ms": 215.45779399999992 - }, - { - "case": "many-short-lines", - "trial": 4, - "mode": "candidate", - "ms": 187.69042199999967 - }, - { - "case": "many-short-lines", - "trial": 5, - "mode": "candidate", - "ms": 194.5716599999996 - }, - { - "case": "many-short-lines", - "trial": 5, - "mode": "baseline", - "ms": 205.19342000000142 - }, - { - "case": "many-short-lines", - "trial": 6, - "mode": "baseline", - "ms": 200.889713999999 - }, - { - "case": "many-short-lines", - "trial": 6, - "mode": "candidate", - "ms": 193.9382970000006 - }, - { - "case": "large-short-label", - "trial": 0, - "mode": "baseline", - "ms": 8.832861999999295 - }, - { - "case": "large-short-label", - "trial": 0, - "mode": "candidate", - "ms": 7.626137999999628 - }, - { - "case": "large-short-label", - "trial": 1, - "mode": "candidate", - "ms": 8.024664000000485 - }, - { - "case": "large-short-label", - "trial": 1, - "mode": "baseline", - "ms": 8.374761000000944 - }, - { - "case": "large-short-label", - "trial": 2, - "mode": "baseline", - "ms": 8.299746999999115 - }, - { - "case": "large-short-label", - "trial": 2, - "mode": "candidate", - "ms": 7.570765000000392 - }, - { - "case": "large-short-label", - "trial": 3, - "mode": "candidate", - "ms": 7.739359000001059 - }, - { - "case": "large-short-label", - "trial": 3, - "mode": "baseline", - "ms": 8.44199000000117 - }, - { - "case": "large-short-label", - "trial": 4, - "mode": "baseline", - "ms": 8.244589999998425 - }, - { - "case": "large-short-label", - "trial": 4, - "mode": "candidate", - "ms": 7.600515000000087 - }, - { - "case": "large-short-label", - "trial": 5, - "mode": "candidate", - "ms": 8.456828999998834 - }, - { - "case": "large-short-label", - "trial": 5, - "mode": "baseline", - "ms": 8.454603000000134 - }, - { - "case": "large-short-label", - "trial": 6, - "mode": "baseline", - "ms": 8.904308999999557 - }, - { - "case": "large-short-label", - "trial": 6, - "mode": "candidate", - "ms": 7.965303999999378 - }, - { - "case": "large-declaration", - "trial": 0, - "mode": "baseline", - "ms": 43.31495399999949 - }, - { - "case": "large-declaration", - "trial": 0, - "mode": "candidate", - "ms": 35.47860899999978 - }, - { - "case": "large-declaration", - "trial": 1, - "mode": "candidate", - "ms": 35.414584000000104 - }, - { - "case": "large-declaration", - "trial": 1, - "mode": "baseline", - "ms": 46.11522800000057 - }, - { - "case": "large-declaration", - "trial": 2, - "mode": "baseline", - "ms": 36.52931399999943 - }, - { - "case": "large-declaration", - "trial": 2, - "mode": "candidate", - "ms": 34.8059489999996 - }, - { - "case": "large-declaration", - "trial": 3, - "mode": "candidate", - "ms": 35.28219600000011 - }, - { - "case": "large-declaration", - "trial": 3, - "mode": "baseline", - "ms": 38.20857099999921 - }, - { - "case": "large-declaration", - "trial": 4, - "mode": "baseline", - "ms": 38.03440099999898 - }, - { - "case": "large-declaration", - "trial": 4, - "mode": "candidate", - "ms": 34.95650199999909 - }, - { - "case": "large-declaration", - "trial": 5, - "mode": "candidate", - "ms": 34.83934399999998 - }, - { - "case": "large-declaration", - "trial": 5, - "mode": "baseline", - "ms": 37.985978000000614 - }, - { - "case": "large-declaration", - "trial": 6, - "mode": "baseline", - "ms": 37.7010850000006 - }, - { - "case": "large-declaration", - "trial": 6, - "mode": "candidate", - "ms": 34.53734900000018 - }, - { - "case": "chunks", - "trial": 0, - "mode": "baseline", - "ms": 65.19520499999999 - }, - { - "case": "chunks", - "trial": 0, - "mode": "candidate", - "ms": 39.78473800000029 - }, - { - "case": "chunks", - "trial": 1, - "mode": "candidate", - "ms": 37.32188899999892 - }, - { - "case": "chunks", - "trial": 1, - "mode": "baseline", - "ms": 67.45899199999985 - }, - { - "case": "chunks", - "trial": 2, - "mode": "baseline", - "ms": 62.80641800000012 - }, - { - "case": "chunks", - "trial": 2, - "mode": "candidate", - "ms": 39.00661599999876 - }, - { - "case": "chunks", - "trial": 3, - "mode": "candidate", - "ms": 39.36458599999969 - }, - { - "case": "chunks", - "trial": 3, - "mode": "baseline", - "ms": 64.03954599999997 - }, - { - "case": "chunks", - "trial": 4, - "mode": "baseline", - "ms": 65.2007450000001 - }, - { - "case": "chunks", - "trial": 4, - "mode": "candidate", - "ms": 37.91183699999965 - }, - { - "case": "chunks", - "trial": 5, - "mode": "candidate", - "ms": 37.205321999999796 - }, - { - "case": "chunks", - "trial": 5, - "mode": "baseline", - "ms": 62.19722600000023 - }, - { - "case": "chunks", - "trial": 6, - "mode": "baseline", - "ms": 63.657884000000195 - }, - { - "case": "chunks", - "trial": 6, - "mode": "candidate", - "ms": 38.11313700000028 - } - ] -} diff --git a/gitnexus/bench/analyze-resume/text-streaming.mjs b/gitnexus/bench/analyze-resume/text-streaming.mjs deleted file mode 100644 index 70f9f2bfe..000000000 --- a/gitnexus/bench/analyze-resume/text-streaming.mjs +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env node -// Compare identical text and chunk outputs against an unchanged built checkout. -// node --expose-gc bench/analyze-resume/text-streaming.mjs /path/to/baseline/gitnexus/dist -import assert from 'node:assert/strict'; -import path from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { createHash } from 'node:crypto'; -import * as candidateText from '../../dist/core/embeddings/embedding-pipeline.js'; -import * as candidateChunks from '../../dist/core/embeddings/character-chunk.js'; - -if (!process.argv[2]) throw new Error('Pass the unchanged baseline dist directory'); -const baselineDirectory = path.resolve(process.argv[2]); -const baselineText = await import( - pathToFileURL(path.join(baselineDirectory, 'core/embeddings/embedding-pipeline.js')) -); -const baselineChunks = await import( - pathToFileURL(path.join(baselineDirectory, 'core/embeddings/character-chunk.js')) -); -const node = (label, content) => ({ - id: `${label}:source.ts:Example`, - name: 'Example', - filePath: 'source.ts', - label, - content, - startLine: 1, - endLine: 1, -}); -const cases = [ - { - name: 'short-code', - node: node('Function', 'export function example() { return "漢😀"; }\n'), - repetitions: 10000, - }, - { name: 'many-short-lines', node: node('File', 'a \t\r\n\n\n'.repeat(600000)), repetitions: 1 }, - { - name: 'large-short-label', - node: node('TypeAlias', 'type Huge = ' + '"漢😀" | '.repeat(300000) + 'never;\n'), - repetitions: 1, - }, - { - name: 'large-declaration', - node: node( - 'Class', - 'export class Example {\n' + - ' field: string;\n'.repeat(100000) + - ' method() { return "😀"; }\n}\n', - ), - repetitions: 1, - }, -]; -const samples = []; -const fingerprints = {}; -for (const entry of cases) { - for (let trial = -1; trial < 7; trial++) { - for (const mode of trial % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { - global.gc?.(); - const impl = mode === 'candidate' ? candidateText : baselineText; - const start = performance.now(); - let fingerprint; - for (let i = 0; i < entry.repetitions; i++) fingerprint = impl.contentHashForNode(entry.node); - const ms = performance.now() - start; - if (fingerprints[entry.name]) assert.equal(fingerprint, fingerprints[entry.name]); - fingerprints[entry.name] = fingerprint; - if (trial >= 0) samples.push({ case: entry.name, trial, mode, ms }); - } - } -} -const content = 'a\n'.repeat(2 * 1024 * 1024); -for (let trial = -1; trial < 7; trial++) { - for (const mode of trial % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { - global.gc?.(); - const impl = mode === 'candidate' ? candidateChunks : baselineChunks; - const start = performance.now(); - const chunks = - mode === 'candidate' - ? impl.iterateCharacterChunks(content, 1, 2097153, 1200, 120) - : impl.characterChunk(content, 1, 2097153, 1200, 120); - const hash = createHash('sha256'); - for (const chunk of chunks) hash.update(JSON.stringify(chunk)); - const ms = performance.now() - start; - const fingerprint = hash.digest('hex'); - if (fingerprints.chunks) assert.equal(fingerprint, fingerprints.chunks); - fingerprints.chunks = fingerprint; - if (trial >= 0) samples.push({ case: 'chunks', trial, mode, ms }); - } -} -const ratios = {}; -for (const name of Object.keys(fingerprints)) { - const median = (mode) => - samples - .filter((s) => s.case === name && s.mode === mode) - .map((s) => s.ms) - .sort((a, b) => a - b)[3]; - ratios[name] = median('candidate') / median('baseline'); -} -process.stdout.write( - JSON.stringify( - { node: process.version, platform: process.platform, fingerprints, ratios, samples }, - null, - 2, - ) + '\n', -); diff --git a/gitnexus/bench/analyze-resume/vector-store-baseline.json b/gitnexus/bench/analyze-resume/vector-store-baseline.json deleted file mode 100644 index 092f73fed..000000000 --- a/gitnexus/bench/analyze-resume/vector-store-baseline.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "node": "v22.23.3", - "platform": "linux", - "nodes": 512, - "dimensions": 256, - "unitNodes": 64, - "samples": [ - { - "trial": 0, - "mode": "baseline", - "ms": 3712.8183809999996 - }, - { - "trial": 0, - "mode": "durable", - "ms": 1684.6306610000001 - }, - { - "trial": 1, - "mode": "durable", - "ms": 1745.7803060000006 - }, - { - "trial": 1, - "mode": "baseline", - "ms": 3620.8808309999995 - }, - { - "trial": 2, - "mode": "baseline", - "ms": 3750.5283279999985 - }, - { - "trial": 2, - "mode": "durable", - "ms": 1692.780707 - }, - { - "trial": 3, - "mode": "durable", - "ms": 2035.2204090000014 - }, - { - "trial": 3, - "mode": "baseline", - "ms": 3115.5813979999984 - }, - { - "trial": 4, - "mode": "baseline", - "ms": 3907.7676600000013 - }, - { - "trial": 4, - "mode": "durable", - "ms": 1761.2801880000006 - } - ], - "ratio": 0.4702035291933125 -} diff --git a/gitnexus/bench/analyze-resume/vector-store.mjs b/gitnexus/bench/analyze-resume/vector-store.mjs deleted file mode 100644 index 4fa174de1..000000000 --- a/gitnexus/bench/analyze-resume/vector-store.mjs +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env node -// Actual vector-store/journal write cost, excluding model and graph ingestion. -// npm run build && node bench/analyze-resume/vector-store.mjs --check -import assert from 'node:assert/strict'; -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; -import lbug from '@ladybugdb/core'; -import { VectorStore } from '../../dist/core/lbug/vector-store.js'; -import { EmbeddingJobJournal } from '../../dist/storage/embedding-job-journal.js'; -import { openLbugConnection, closeLbugConnection } from '../../dist/core/lbug/lbug-config.js'; - -process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; -const nodes = 512; -const dimensions = 256; -const unitNodes = 64; -const identity = 'a'.repeat(64); -const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-measure-')); -const samples = []; -const manifest = { - version: 1, - sourceIdentity: identity, - digest: 'b'.repeat(64), - bytes: nodes / unitNodes, - totalNodes: nodes, - totalUnits: nodes / unitNodes, - maximumUnitBytes: 1024, -}; -const chunk = (node) => ({ - nodeId: `Function:source.ts:node${node}`, - contentHash: 'same-input-hash', - chunkIndex: 0, - startLine: 1, - endLine: 2, - embedding: Array(dimensions).fill(0.125), -}); - -async function baseline(directory) { - const handle = await openLbugConnection(lbug, path.join(directory, 'vectors'), { - bufferPoolBytes: 64 * 1024 * 1024, - threads: 1, - }); - const prepared = new Map(); - async function query(sql, params) { - let result; - if (params) { - let statement = prepared.get(sql); - if (!statement) { - statement = await handle.conn.prepare(sql); - prepared.set(sql, statement); - } - result = await handle.conn.execute(statement, params); - } else result = await handle.conn.query(sql); - const parts = Array.isArray(result) ? result : [result]; - const rows = []; - try { - for (const part of parts) while (part.hasNext()) rows.push(part.getNextSync()); - } finally { - for (const part of parts) part.close(); - } - return rows; - } - try { - await query( - `CREATE NODE TABLE CodeEmbedding(id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${dimensions}], contentHash STRING, PRIMARY KEY(id))`, - ); - const start = performance.now(); - // The existing batchInsertEmbeddings path: prepared DELETE then CREATE, - // with each write auto-committed and a checkpoint after the full pipeline. - for (let offset = 0; offset < nodes; offset += unitNodes) { - const rows = Array.from({ length: unitNodes }, (_, i) => chunk(offset + i)); - for (const row of rows) - await query('MATCH (e:CodeEmbedding {id:$id}) DELETE e', { id: `${row.nodeId}:0` }); - for (const row of rows) - await query( - 'CREATE (:CodeEmbedding {id:$id, nodeId:$nodeId, chunkIndex:$chunkIndex, startLine:$startLine, endLine:$endLine, embedding:$embedding, contentHash:$contentHash})', - { id: `${row.nodeId}:0`, ...row }, - ); - } - await query('CHECKPOINT'); - const ms = performance.now() - start; - assert.equal(Number((await query('MATCH (e:CodeEmbedding) RETURN count(e) AS n'))[0].n), nodes); - return ms; - } finally { - await closeLbugConnection(handle); - } -} - -async function durable(directory) { - const store = await VectorStore.open({ - directory, - identity, - dimensions, - bufferPoolBytes: 64 * 1024 * 1024, - threads: 1, - batchBytes: 1024 * 1024, - mode: 'create', - }); - try { - const start = performance.now(); - const journal = await EmbeddingJobJournal.create(directory, identity, manifest); - for (let offset = 0; offset < nodes; offset += unitNodes) { - const ordinal = offset / unitNodes; - const rows = Array.from({ length: unitNodes }, (_, i) => chunk(offset + i)); - await journal.begin({ - start: { unit: ordinal, offset: ordinal }, - next: { unit: ordinal + 1, offset: ordinal + 1 }, - nodes: unitNodes, - origin: 'manifest', - }); - await store.beginNodes(rows); - await store.appendBatch(rows); - const receipts = await store.finishNodes( - rows.map(({ nodeId, contentHash }) => ({ nodeId, contentHash, chunks: 1 })), - ); - await store.checkpointUnit(receipts); - await journal.acknowledge({ reusedNodes: 0, chunks: unitNodes }); - } - const ms = performance.now() - start; - const state = journal.snapshot(); - assert.equal(state.completedNodes, nodes); - assert.equal(state.chunks, nodes); - assert.equal(state.active, null); - assert.equal(await store.seal(nodes, nodes), 'exact'); - return ms; - } finally { - await store.close(); - } -} - -try { - for (let trial = 0; trial < 5; trial++) { - for (const mode of trial % 2 ? ['durable', 'baseline'] : ['baseline', 'durable']) { - const directory = path.join(root, `${trial}-${mode}`); - await fs.mkdir(directory); - samples.push({ - trial, - mode, - ms: await (mode === 'baseline' ? baseline : durable)(directory), - }); - } - } - const median = (mode) => - samples - .filter((s) => s.mode === mode) - .map((s) => s.ms) - .sort((a, b) => a - b)[2]; - const ratio = median('durable') / median('baseline'); - process.stdout.write( - JSON.stringify( - { - node: process.version, - platform: process.platform, - nodes, - dimensions, - unitNodes, - samples, - ratio, - }, - null, - 2, - ) + '\n', - ); - if (process.argv.includes('--check')) - assert.ok(ratio <= 1, `Durable native writes regressed: ${ratio.toFixed(3)}x`); -} finally { - await fs.rm(root, { recursive: true, force: true }); -} From 0e0329c1d46f5d07429b9bacb1fe61bac86e3a5d Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 15:04:01 +0000 Subject: [PATCH 43/67] fix(analyze): close recovery and CI gaps Retain ownership receipts and paid vectors through interrupted work and cleanup. Restore automatic heap and parser-worker sizing, and cover publication, cancellation and capacity failures. Refs #3829, #3598 Note: pre-existing setup.test.ts and setup-skills.test.ts failures with inherited CODEX_HOME; all 62 tests pass with it unset for the test process. --- .github/workflows/ci-tests.yml | 3 +- ARCHITECTURE.md | 36 ++- GUARDRAILS.md | 8 +- MIGRATION.md | 31 +++ docs/cli-reference.md | 31 ++- gitnexus/README.md | 7 +- gitnexus/bench/analyze-resume/README.md | 15 +- gitnexus/bench/analyze-resume/measure.mjs | 115 +++++++-- gitnexus/scripts/cross-platform-tests.ts | 14 ++ gitnexus/src/cli/analyze-watch.ts | 12 +- gitnexus/src/cli/analyze.ts | 31 ++- gitnexus/src/cli/help-i18n.ts | 1 + gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 2 + gitnexus/src/core/analysis-embedding-phase.ts | 6 +- .../src/core/analysis-graph-verification.ts | 8 +- gitnexus/src/core/analysis-host-resources.ts | 11 +- .../src/core/analysis-phase-supervisor.ts | 33 --- gitnexus/src/core/analysis-progress.ts | 10 +- gitnexus/src/core/analysis-publication.ts | 69 +++++- gitnexus/src/core/analysis-resource-budget.ts | 91 ++++--- gitnexus/src/core/embeddings/chunker.ts | 30 +-- .../core/embeddings/embedding-job-runner.ts | 32 ++- gitnexus/src/core/embeddings/embedding-job.ts | 10 + .../src/core/embeddings/embedding-sidecar.ts | 4 + gitnexus/src/core/embeddings/http-client.ts | 13 +- .../embeddings/vector-generation-search.ts | 11 +- .../src/core/embeddings/vector-publication.ts | 8 +- .../ingestion/pipeline-phases/parse-impl.ts | 9 +- .../core/ingestion/pipeline-phases/runner.ts | 1 + gitnexus/src/core/ingestion/pipeline.ts | 4 + .../src/core/ingestion/utils/effective-ram.ts | 2 +- .../src/core/ingestion/workers/worker-pool.ts | 10 +- gitnexus/src/core/lbug/vector-store.ts | 52 +++- gitnexus/src/core/run-analyze.ts | 26 +- gitnexus/src/core/shared-store-analyze.ts | 3 +- gitnexus/src/mcp/local/local-backend.ts | 3 + gitnexus/src/server/analyze-job.ts | 35 ++- gitnexus/src/server/api.ts | 16 +- .../src/storage/analysis-artifact-cleanup.ts | 120 +++++++++ gitnexus/src/storage/embedding-artifact.ts | 41 +++ gitnexus/src/storage/embedding-job-journal.ts | 52 +++- .../src/storage/embedding-job-progress.ts | 16 +- gitnexus/src/storage/index-capabilities.ts | 9 +- gitnexus/src/storage/index-lock.ts | 4 +- gitnexus/src/storage/repo-meta.ts | 3 + gitnexus/src/storage/vector-generation.ts | 4 + .../test/helpers/analysis-process-exit.ts | 31 +++ .../test/helpers/prepared-analysis-runner.mjs | 8 +- .../analysis-embedding-flow.test.ts | 20 +- .../analysis-publication-flow.test.ts | 22 +- .../embedding-job-controller.test.ts | 25 +- .../embedding-vector-store.test.ts | 201 ++++++++++++++- .../legacy-embedding-controller.test.ts | 7 +- .../integration/legacy-vector-source.test.ts | 49 ++++ .../test/unit/analysis-host-resources.test.ts | 14 +- .../unit/analysis-phase-supervisor.test.ts | 2 +- .../test/unit/analysis-publication.test.ts | 233 +++++++++++++++++- .../unit/analysis-resource-budget.test.ts | 44 +++- .../test/unit/analyze-heap-respawn.test.ts | 22 +- gitnexus/test/unit/analyze-job.test.ts | 41 ++- gitnexus/test/unit/api-fts-mode.test.ts | 6 +- gitnexus/test/unit/chunker.test.ts | 34 ++- .../test/unit/embedding-job-journal.test.ts | 54 ++++ .../test/unit/embedding-job-progress.test.ts | 26 ++ .../unit/embedding-sidecar-lifetime.test.ts | 30 +++ .../unit/embedding-streaming-text.test.ts | 10 +- .../test/unit/embeddings-sync-command.test.ts | 7 +- .../http-embedding-response-budget.test.ts | 13 + gitnexus/test/unit/index-lock.test.ts | 12 + .../test/unit/lbug-resource-release.test.ts | 2 +- .../local-backend-vector-generation.test.ts | 8 + .../test/unit/parse-failure-drain.test.ts | 47 ++++ gitnexus/test/unit/pipeline-runner.test.ts | 25 ++ gitnexus/test/unit/watch-cancellation.test.ts | 5 +- .../test/unit/watch-external-config.test.ts | 5 +- .../unit/watch-supervisor-reservation.test.ts | 34 ++- .../unit/worker-pool-timeout-retire.test.ts | 55 +++++ gitnexus/vitest.config.ts | 12 + 79 files changed, 1892 insertions(+), 266 deletions(-) create mode 100644 gitnexus/src/storage/analysis-artifact-cleanup.ts create mode 100644 gitnexus/test/helpers/analysis-process-exit.ts create mode 100644 gitnexus/test/unit/embedding-sidecar-lifetime.test.ts create mode 100644 gitnexus/test/unit/parse-failure-drain.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5373d9a74..940ad0900 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -60,7 +60,8 @@ jobs: if [[ "$RESOURCE_PROFILE" == default ]]; then resource_args+=(--default-resources) fi - node bench/analyze-resume/measure.mjs --check \ + node bench/analyze-resume/measure.mjs --validate-gate + node bench/analyze-resume/measure.mjs --check --allow-dependency-changes \ --baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \ --output "$RUNNER_TEMP/analyze-resume/results.json" \ "${resource_args[@]}" 2>&1 | tee "$RUNNER_TEMP/analyze-resume/run.log" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0d9e32b18..8c4d30b67 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -640,25 +640,31 @@ Inheritance is captured by the `@reference.inherits` tag and emitted by the scop ``` CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) - 1. Early exit if lastCommit == HEAD (unless --force) [0%] - 2. Cache existing embeddings from prior index [0%] - 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] - 4. Clean up legacy KuzuDB files [60%] - 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] - 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] - 7. Restore cached embeddings (batch insert) [88%] - 8. Generate new embeddings if --embeddings [90-98%] - 9. Save metadata + register repo + update .gitignore [98-100%] - 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] + 1. Admit phase resources and recover any prepared graph publication. + 2. Reuse unchanged graph/parser work where its receipts still match. + 3. runPipelineFromRepo() → KnowledgeGraph → bounded graph write → FTS. + 4. Prepare the owned graph and publication receipt; drain native resources. + 5. Supervisor waits for graph-process exit, verifies and publishes metadata. + 6. If embeddings are requested/pending, open the resumable embedding job. + 7. Reuse verified vectors, checkpoint bounded units, then seal/publish vectors. + 8. Register the repo and generate AI context files. + +Graph-only legacy slots keep their existing path until separate publication is +needed. A ready graph remains queryable while an embedding job is pending; +semantic search reports degradation until a compatible sealed artifact is ready. ``` -**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. +**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, resumable), `--skipGit`, `--noStats`. ## Storage ``` /.gitnexus/ - ├── lbug # LadybugDB database + ├── lbug # Legacy monolithic database, until first separate publication + ├── lbug.generation. # Owned private graph, selected by graphPath in metadata + ├── analysis-publication.json # Recoverable prepared graph receipt + ├── embeddings// # Job journal, manifest and separate vector database + ├── embeddings/spaces/ # Model-space ownership/reuse receipts ├── lbug.wal # Write-ahead log ├── lbug.shadow # Shadow sidecar (checkpoint staging) ├── lbug.lock # Single-writer lock @@ -690,7 +696,11 @@ the read-only open is retried — see `sidecar-recovery.ts` (`isReadOnlyCheckpointInProgressError`) and the `lbug-interrupted-checkpoint-recovery` integration test. -Managed by `repo-manager.ts`. +Managed by `repo-manager.ts`. Publication reclaims unreferenced private graphs +and job artifacts under the slot lock, preserving current jobs, model spaces, +legacy reuse sources and prepared publications. Busy reader files are retried on +a later publication. See [MIGRATION.md](MIGRATION.md#resumable-analysis-and-separate-vector-storage-3598) +before mixing binary versions or rolling back. ### Automatic parse-cache identity diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 82896c508..8750bc69c 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -44,13 +44,13 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Trigger:** Semantic search quality drops; `stats.embeddings` in the index metadata (`gitnexus.json` / legacy `meta.json`) is 0 after refresh. - **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected. -- **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. +- **Why:** Plain `analyze` preserves prior vectors in their artifact (or restores the legacy in-graph cache); ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. ### Analyze finishes but embeddings are incomplete (partial embedding index) -- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["embedding-checkpoint-pending"]` (or the human-readable "Index incomplete reasons" line); `stats.embeddings` is honest and **non-zero**, and the preceding analyze log showed a `Warning: N node(s) lost their embeddings to embedding-endpoint failures` line (#2790). -- **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. -- **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. +- **Trigger:** `gitnexus status --json` reports `embedding-job-pending` (or the legacy `embedding-checkpoint-pending`), or an embedding failure follows graph publication. +- **Do:** Re-run plain `gitnexus analyze`; retained work resumes without `--embeddings`. Inspect `embeddingProgress` for completed, reused, failed and excluded nodes. `--force` rebuilds the graph and retains compatible vectors for reuse; `--drop-embeddings` deliberately abandons embedding work. +- **Why:** Graph and vector publication are separate. The graph is usable while a job is pending; semantic search reports degradation until a sealed compatible artifact is published. Deterministic input rejections are retried across at most three invocations, then excluded with a recorded reason. Provider outages remain resumable. Do not delete job artifacts to clear a status warning: they contain paid vectors. ### Scope extraction is incomplete diff --git a/MIGRATION.md b/MIGRATION.md index 36a899840..487684443 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,5 +1,36 @@ # Migration Guide +## Resumable analysis and separate vector storage (#3598) + +Analysis with embeddings publishes a private `lbug.generation.` graph and +stores resumable vector artifacts under `embeddings//`. Metadata names +the active graph and sealed vector generation. Once a slot uses this layout, +plain analysis, watch and hooks continue using it. Pending embedding work resumes +on the next plain `gitnexus analyze`; `--force` rebuilds the graph while retaining +compatible vectors for reuse. `--drop-embeddings` explicitly abandons embeddings. + +Upgrade every CLI, MCP/serve process and installed hook that accesses the slot. +Restart long-running clients and rerun `gitnexus setup` with the upgraded binary. +Older versions do not understand the private graph pointer or separate vectors. +Do not run an older analyzer against the migrated slot. + +Before upgrading, stop readers and writers and back up the complete storage +slot, including metadata, graph files, `embeddings/`, and any referenced shared +store. To roll back, stop clients again and restore that complete backup with +the matching older binaries. Renaming a generation file to `lbug` does not move +vectors back into it. There is no reverse conversion for separate vector +artifacts; without a backup, rebuilding with an older version requires generating +embeddings again and can incur provider charges. + +A graph can be ready while embeddings remain pending. Embedding failure after +graph publication exits nonzero; the graph remains usable and the next analyze +resumes paid work. HTTP clients should read `embeddingProgress` and capability +reasons rather than relying on the former terminal SSE `partial` field. After +three deterministic input rejections, the job publishes healthy nodes and records +`excludedNodes` with reason `input-rejected`; status reports the omission. +On Windows, a legacy reader can prevent retirement of `lbug`. Stop that reader +and rerun analyze; the prepared publication and vector artifacts are retained. + ## LadybugDB 0.22.0 storage and extensions LadybugDB 0.22.0 reads indexes written by 0.21.1, but a write/checkpoint upgrades diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 524fbd1ab..b6fd91bab 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -102,6 +102,7 @@ gitnexus analyze --max-processes # Process-detection process cap (replaces gitnexus analyze --max-entry-point-candidates # Ranked entry-point pool (default 200; raise when the warning names it) gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots gitnexus analyze --asyncapi-spec ./docs/asyncapi # Resolve broker addresses from AsyncAPI 3.x documents +gitnexus analyze --resource-budget 8192 # Strict process-family reservation in MiB gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and --max-old-space-size gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) @@ -284,7 +285,7 @@ For local model setup and switching instructions, see [Local Embedding Model](.. | `GITNEXUS_CONTENT_RETENTION` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | You need to reduce persisted source text while preserving graph structure. | | `GITNEXUS_SKIP_FTS` | unset | When exactly `1`, skips FTS extension loading and keyword index creation during analyze. Equivalent to `--skip-fts`; a later analyze without either option restores FTS. | Graph-only consumers with their own retrieval, or short-lived indexes that do not need keyword search. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded analysis requires a positive byte ceiling of at least 8 MiB; zero and malformed values fail preflight. Other readers retain the legacy zero/default policy. During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning, and invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | | `GITNEXUS_ATOMIC_INCREMENTAL` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. | A staged incremental analyze fails (see _Troubleshooting_ in the package README), or you need in-place writes. | | `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | @@ -381,3 +382,31 @@ flowchart TD ConnA -->|"queries"| RepoA ConnB -->|"queries"| RepoB ``` + +### Analysis resource admission and resumable status + +Automatic analysis retains the RAM-aware main heap and independent parser-worker +ceilings. These are virtual limits and may sum above physical RAM. Use +`--resource-budget ` for a strict phase reservation covering the supervisor, +main heap, workers, native pools, queues and embedding model. The two phases do +not overlap. `--memory-budget` alone changes the main heap without removing parser +workers; with a strict total budget, incompatible overrides fail before work. +Embedding thread settings are validated only when embeddings are requested or pending. + +`status --json` reports `index.graphStatus` for a checkout, +`index.graphIncompleteReasons`, `index.embeddingProgress` and `index.capabilities`. +Progress includes its phase, completed/reused/failed nodes and pending units; +`excludedNodes` records permanently rejected inputs. Stored metadata separately +names `graphGeneration`, `vectorGeneration` and `embeddingJob`. +A preserved vector generation may refer to an older graph. Pending jobs report +`embedding-job-pending`; a published artifact with omitted inputs reports +`embedding-inputs-rejected`. A ready graph does not imply ready semantic search. + +`GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` is an internal inherited reservation, in +nonnegative integer bytes, for the waiting parent process. Leave it unset for +automatic accounting; a supplied value is included in admission. It does not +set the parent's V8 heap limit. + +A local-model job can report "Embedding runtime is busy" while an existing query +owns the model. Retry after that query finishes; admission does not replace a +sidecar that still has physical requests in flight. diff --git a/gitnexus/README.md b/gitnexus/README.md index 3b683d431..64de67835 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -779,7 +779,7 @@ Configure the behavior with these environment variables: | `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. | | `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores the native 80%-of-RAM default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | Native buffer-pool ceiling in bytes. Bounded analysis requires a positive integer of at least 8 MiB; zero and malformed values fail preflight. Query readers retain the legacy zero/default policy. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning. Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | | `GITNEXUS_ATOMIC_INCREMENTAL` | `0`, `1` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. | @@ -846,7 +846,9 @@ only; parse workers keep their own caps. Use `--resource-budget ` to admit the complete analyzer process family, including the frontend, main heap, workers, native pools, embedding model, input queue, and headroom. An explicit heap or pool that cannot fit is rejected before -work starts. Defaults derive from available memory and CPUs; this reservation is +work starts. Automatic sizing retains independent RAM-aware main and worker +ceilings, whose sum may exceed RAM. Only an explicit resource budget imposes an +aggregate phase ceiling; this reservation is not an operating-system RSS limit. `analyze --watch` accepts the same budget. Embedding-enabled analysis publishes the graph and keyword index first. If the @@ -974,6 +976,7 @@ Four env vars expose the pool's resilience layers (respawn budget, cumulative-ti | `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. | | `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. | +| `GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` | nonnegative integer bytes | automatic | Internal inherited reservation for the waiting parent process. Leave unset for automatic accounting; a supplied value counts toward analysis admission and does not set the parent heap. | | `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. | | `GITNEXUS_HEAP_LIMIT_SOURCE` | unset | Set by analyze itself (`budget` or `auto`) to record where the heap limit came from, so out-of-memory advice points at `--memory-budget` rather than a `--max-old-space-size` pin. | Never — internal; set `--memory-budget` instead. | | `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. | diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 8b5f1c052..86221d0af 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -22,18 +22,27 @@ and candidate in alternating order, after a warmup, on the current machine. The six CI scenarios cover graph-only analysis, warm no-op, an incremental edit, fresh HTTP embeddings, cached HTTP embeddings, and retry after a deterministic provider outage. Setup runs are outside the measured interval. Assertions require -equal graph/vector counts and embedding-input fingerprints, matching runtime +equal graph/vector counts and embedding-input fingerprints, recorded runtime dependencies, completed publication, and no paid requests for a cached run. `--check` requires all six scenarios, the recorded corpus size, a warmup and at -least seven pairs. It fails when the paired 95% bootstrap interval lies entirely -above the candidate/base ratio of **1.0**. An interval crossing 1 remains +least seven pairs. It fails when the paired 95% interval lies entirely +above the candidate/base ratio of **1.0**. The interval encloses both the +percentile bootstrap and Student-t interval on paired log ratios, avoiding the +narrow bootstrap-only interval with seven samples. Student-t inference assumes +independent pairs with approximately normal log ratios +([NIST confidence interval definition](https://www.itl.nist.gov/div898/handbook/eda/section3/eda352.htm)). An interval crossing 1 remains **inconclusive** and is reported as such; a successful CI gate means no slowdown was statistically detected, not proof of equal performance. The original graph-only interval in `baseline.json` is inconclusive and has not been discarded. `.github/workflows/ci-tests.yml` runs both resource profiles against the PR base (the first parent on non-PR runs). Failures block the existing CI Gate. Each job +uses `--allow-dependency-changes` to measure the complete shipped runtimes, so +dependency updates and release-version changes are benchmarked as well. Reports +record both lockfile/native digests and each pair's resolved dependency receipts +and parity. Manual runs require identical dependencies unless this flag is set. +Each job runs `--validate-gate` decision checks before measuring and uploads raw samples, summaries, dependency receipts and analyzer logs, including partial results on failure. The runner never overwrites an existing report. diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs index 74728421c..e8e4dcec5 100644 --- a/gitnexus/bench/analyze-resume/measure.mjs +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -40,7 +40,7 @@ async function runtimeEnvironment(cli) { }; } -export function assertComparableRuntimes(baseline, candidate) { +export function assertComparableRuntimes(baseline, candidate, allowDependencyChanges = false) { for (const component of ['cli', 'nativeEntry']) { assert.equal( candidate[component].device, @@ -53,12 +53,14 @@ export function assertComparableRuntimes(baseline, candidate) { `${component} runtime files are on different filesystem types`, ); } - assert.equal(candidate.lockDigest, baseline.lockDigest, 'Dependency lockfiles differ'); - assert.equal( - candidate.nativeEntryDigest, - baseline.nativeEntryDigest, - 'Native runtime entry bytes differ', - ); + if (!allowDependencyChanges) { + assert.equal(candidate.lockDigest, baseline.lockDigest, 'Dependency lockfiles differ'); + assert.equal( + candidate.nativeEntryDigest, + baseline.nativeEntryDigest, + 'Native runtime entry bytes differ', + ); + } } export function assertCompletedCase(scenario, metadata, beforeMetadata) { @@ -88,7 +90,7 @@ const median = (values) => { return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; }; -/** Paired log-ratio bootstrap. An interval crossing 1 is unresolved, never a pass. */ +/** Paired log-ratio intervals. Crossing 1 is unresolved, never equivalence. */ export function summarizePairs(pairs) { assert(pairs.length > 0, 'At least one measured pair is required'); const logs = pairs.map(({ baseline, candidate }) => { @@ -106,13 +108,42 @@ export function summarizePairs(pairs) { const samples = Array.from({ length: 10_000 }, () => Math.exp(average(logs.map(() => logs[Math.floor(random() * logs.length)]))), ).sort((a, b) => a - b); - const interval = [samples[250], samples[9749]]; + const bootstrap = [samples[250], samples[9749]]; + // Bootstrap alone understates uncertainty with seven pairs. Enclose it in + // the Student-t interval for the mean log ratio; round df down in this table. + const df = logs.length - 1; + const critical = [ + [1, 12.707], + [2, 4.303], + [3, 3.183], + [4, 2.777], + [5, 2.571], + [6, 2.447], + [7, 2.365], + [8, 2.307], + [9, 2.263], + [10, 2.229], + [15, 2.132], + [20, 2.086], + [30, 2.043], + ] + .filter(([degree]) => degree <= df) + .at(-1)?.[1]; + const mean = average(logs); + const margin = + df > 0 + ? critical * Math.sqrt(logs.reduce((sum, x) => sum + (x - mean) ** 2, 0) / df / logs.length) + : 0; + const student = [Math.exp(mean - margin), Math.exp(mean + margin)]; + const interval = [Math.min(bootstrap[0], student[0]), Math.max(bootstrap[1], student[1])]; return { pairs: pairs.length, baselineMedianMs: median(pairs.map((p) => p.baseline)), candidateMedianMs: median(pairs.map((p) => p.candidate)), geometricRatio: Math.exp(average(logs)), - pairedBootstrap95: interval, + pairedBootstrap95: bootstrap, + pairedStudent95: student, + paired95: interval, evidence: pairs.length < 7 ? 'smoke-only' @@ -135,7 +166,7 @@ export function checkPerformance(summaries, baseline) { for (const scenario of scenarios) { const summary = summaries[scenario]; assert(summary && summary.pairs >= 7, `Missing seven measured pairs for ${scenario}`); - const interval = summary.pairedBootstrap95; + const interval = summary.paired95; assert( Array.isArray(interval) && interval.length === 2 && @@ -485,12 +516,54 @@ export async function main(argv = process.argv.slice(2)) { 'heap-mb': { type: 'string', default: '2048' }, workers: { type: 'string', default: '2' }, 'default-resources': { type: 'boolean', default: false }, + 'allow-dependency-changes': { type: 'boolean', default: false }, 'sample-ms': { type: 'string', default: '25' }, 'keep-stores': { type: 'boolean', default: false }, check: { type: 'boolean', default: false }, + 'validate-gate': { type: 'boolean' }, help: { type: 'boolean' }, }, }); + if (v['validate-gate']) { + const fixture = { schemaVersion: 1, maxPairedRatio: 1, scenarios: { case: {} } }; + const summary = (ratios) => + summarizePairs(ratios.map((candidate) => ({ baseline: 1, candidate }))); + assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(1.1)) }, fixture), { + failedScenarios: ['case'], + inconclusiveScenarios: [], + }); + assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(0.9)) }, fixture), { + failedScenarios: [], + inconclusiveScenarios: [], + }); + const mixed = summary([0.8, 0.9, 0.95, 1, 1.05, 1.1, 1.2]); + assert.deepEqual(checkPerformance({ case: mixed }, fixture), { + failedScenarios: [], + inconclusiveScenarios: ['case'], + }); + assert(mixed.paired95[0] <= mixed.pairedBootstrap95[0]); + assert(mixed.paired95[1] >= mixed.pairedBootstrap95[1]); + assert.throws(() => checkPerformance({ case: summary([1, 1]) }, fixture)); + assert.throws(() => summarizePairs([{ baseline: 0, candidate: 1 }])); + const runtime = { + cli: { device: '1', filesystem: '1' }, + nativeEntry: { device: '1', filesystem: '1' }, + lockDigest: 'a', + nativeEntryDigest: 'a', + }; + const changed = { ...runtime, lockDigest: 'b', nativeEntryDigest: 'b' }; + assert.throws(() => assertComparableRuntimes(runtime, changed)); + assertComparableRuntimes(runtime, changed, true); + assert.throws(() => + assertComparableRuntimes( + runtime, + { ...changed, cli: { device: '2', filesystem: '1' } }, + true, + ), + ); + console.log('Paired gate decision checks passed'); + return; + } if (v.help) { console.log(`Usage: node bench/analyze-resume/measure.mjs --baseline /repo/gitnexus/dist/cli/index.js --output /tmp/results.json --candidate PATH Defaults to this checkout's built CLI @@ -498,6 +571,8 @@ export async function main(argv = process.argv.slice(2)) { --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 --default-resources Omit heap, worker and native-pool overrides on both sides + --allow-dependency-changes Compare shipped runtimes, including intentional dependency changes + --validate-gate Check decision logic without running benchmarks --check Require the baseline corpus/scenarios and reject detected slowdowns --keep-stores Retain owned index/home artifacts (logs and measurements always retained) Includes full CLI startup, graph/manifest/native writes and final index work. @@ -550,7 +625,7 @@ Live-query latency and platform cases remain separate acceptance requirements.`) baseline: await runtimeEnvironment(clis.baseline), candidate: await runtimeEnvironment(clis.candidate), }; - assertComparableRuntimes(runtime.baseline, runtime.candidate); + assertComparableRuntimes(runtime.baseline, runtime.candidate, v['allow-dependency-changes']); const output = path.resolve(v.output); await fs.mkdir(path.dirname(output), { recursive: true }); // Never overwrite a prior result; evidence files and temporary directories are exclusively owned. @@ -578,6 +653,7 @@ Live-query latency and platform cases remain separate acceptance requirements.`) totalMemory: os.totalmem(), settings: numbers, resourceProfile: v['default-resources'] ? 'default' : 'explicit-heap-workers-pool', + comparison: v['allow-dependency-changes'] ? 'shipped-runtimes' : 'identical-dependencies', runtime, clis: {}, }, @@ -663,11 +739,18 @@ Live-query latency and platform cases remain separate acceptance requirements.`) /^sha256:[a-f0-9]{64}$/.test(baselineDependencies ?? ''), 'Baseline dependency receipt missing', ); - assert.equal( - pair.candidate.metadata.runnerIdentity?.dependencyRuntime?.digest, - baselineDependencies, - 'Resolved dependency payloads differ; timings are not comparable', + const candidateDependencies = + pair.candidate.metadata.runnerIdentity?.dependencyRuntime?.digest; + assert( + /^sha256:[a-f0-9]{64}$/.test(candidateDependencies ?? ''), + 'Candidate dependency receipt missing', ); + pair.dependenciesMatch = candidateDependencies === baselineDependencies; + if (!v['allow-dependency-changes']) + assert( + pair.dependenciesMatch, + 'Resolved dependency payloads differ; use --allow-dependency-changes to compare shipped runtimes', + ); if (iteration >= 0) pairs.push({ baseline: pair.baseline.elapsedMs, candidate: pair.candidate.elapsedMs }); } diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 627e86bc7..0e4c92cc5 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -26,6 +26,12 @@ // Platform-specific logic tests — contain explicit process.platform guards // or test behavior that differs across operating systems const PLATFORM_LOGIC = [ + 'test/unit/embedding-sidecar-lifetime.test.ts', + 'test/unit/analysis-publication.test.ts', + 'test/unit/analysis-phase-supervisor.test.ts', + 'test/unit/shared-vector-artifact.test.ts', + 'test/unit/embedding-artifact.test.ts', + 'test/unit/embedding-job-progress.test.ts', // Descriptor validation differs when O_NOFOLLOW is unavailable on Windows. 'test/unit/embedding-job-file-security.test.ts', 'test/unit/shared-store-read-security.test.ts', @@ -167,6 +173,14 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/analysis-embedding-flow.test.ts', + 'test/integration/analysis-generation-routing.test.ts', + 'test/integration/analysis-publication-flow.test.ts', + 'test/integration/embedding-job-controller.test.ts', + 'test/integration/embedding-job-restart.test.ts', + 'test/integration/embedding-native-release.test.ts', + 'test/integration/legacy-embedding-controller.test.ts', + 'test/integration/legacy-vector-source.test.ts', 'test/integration/embedding-vector-store.test.ts', 'test/integration/legacy-vector-reuse-index.test.ts', 'test/integration/legacy-vector-export.test.ts', diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index f4382203d..a281ce9ea 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -24,7 +24,7 @@ import { } from './analyze-config.js'; import { logger } from '../core/logger.js'; import type { AnalyzeOptions } from './analyze-options.js'; -import { ensureHeap } from './analyze.js'; +import { ensureHeap, usesManualAnalysisHeap } from './analyze.js'; import { cliError, cliInfo, cliWarn } from './cli-message.js'; import { parseIntegerOption } from './int-option.js'; import { @@ -495,9 +495,12 @@ export async function watchCommandWithRunnerIdentity( } const repoPath = await fs.realpath(requestedRepoPath); try { + const manualHeap = usesManualAnalysisHeap(cliOptions.memoryBudget); const admission = resolveHostAnalysisResources({ totalMb: positiveInteger(cliOptions.resourceBudget, '--resource-budget'), - mainHeapMb: positiveInteger(cliOptions.memoryBudget, '--memory-budget'), + mainHeapMb: manualHeap + ? Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2) + : positiveInteger(cliOptions.memoryBudget, '--memory-budget'), workers: positiveInteger(cliOptions.workers, '--workers'), includeEmbeddings: await analysisMayEmbed(repoPath, { branch: cliOptions.branch }), supervisorBytes: @@ -507,11 +510,12 @@ export async function watchCommandWithRunnerIdentity( }); if (cancelled) return; if ( - await ensureHeap({ + !manualHeap && + (await ensureHeap({ cleanForwardedTermination: true, memoryBudget: cliOptions.memoryBudget, admittedHeapMb: admission.mainHeapMb, - }) + })) ) return; await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS); diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 4e3fbd6a1..9ae5eda6c 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -424,6 +424,7 @@ const runRespawnedAnalyze = ( let forwardedSignal: NodeJS.Signals | undefined; let prepared: CliPreparedAnalysisMessage | undefined; let phaseError: string | undefined; + let cancelTimer: ReturnType | undefined; const child = spawn(process.execPath, [...args], { stdio: ['inherit', 'pipe', 'pipe', 'ipc'], windowsHide: true, @@ -431,6 +432,11 @@ const runRespawnedAnalyze = ( }); const forwardSignal = (signal: NodeJS.Signals): void => { forwardedSignal ??= signal; + // This is a standalone process, not a native worker thread. A wedged + // process may be killed after grace; dirty/staged receipts recover it. + cancelTimer ??= setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + }, 60_000); if (child.connected) { try { child.send({ type: 'cancel' }, () => { @@ -446,6 +452,7 @@ const runRespawnedAnalyze = ( const finish = (exit: RespawnExit): void => { if (settled) return; settled = true; + clearTimeout(cancelTimer); process.removeListener('SIGINT', forwardSigint); process.removeListener('SIGTERM', forwardSigterm); resolve({ @@ -457,8 +464,8 @@ const runRespawnedAnalyze = ( }); }; - process.once('SIGINT', forwardSigint); - process.once('SIGTERM', forwardSigterm); + process.on('SIGINT', forwardSigint); + process.on('SIGTERM', forwardSigterm); child.on('message', (message: unknown) => { try { @@ -603,6 +610,18 @@ const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024; * when absent/unparseable. Last occurrence wins, matching V8's own * later-flag-wins semantics when NODE_OPTIONS repeats a flag. */ +/** CLI and watch must preserve the same explicit heap/autopilot opt-outs. */ +export function usesManualAnalysisHeap(memoryBudget?: string): boolean { + return ( + memoryBudget === undefined && + (process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP === '1' || + (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && + (memoryAutopilotDisabled() || + parseMaxOldSpaceMb(process.execArgv.join(' ')) !== null || + (parseMaxOldSpaceMb(process.env.NODE_OPTIONS ?? '') ?? 0) >= RESPAWN_HEAP_MB))) + ); +} + export function parseMaxOldSpaceMb(nodeOptions: string): number | null { return parseV8SizeFlagMb(nodeOptions, 'max-old-space-size'); } @@ -1391,13 +1410,7 @@ const analyzeCommandImpl = async ( process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined ? analysisSupervisorReservation() : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES); - manualHeap = - options.memoryBudget === undefined && - (process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP === '1' || - (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && - (memoryAutopilotDisabled() || - parseMaxOldSpaceMb(process.execArgv.join(' ')) !== null || - (parseMaxOldSpaceMb(process.env.NODE_OPTIONS ?? '') ?? 0) >= RESPAWN_HEAP_MB))); + manualHeap = usesManualAnalysisHeap(options.memoryBudget); resources = resolveHostAnalysisResources({ totalMb: options.resourceBudget === undefined diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index 7a3ff451f..85474843d 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -74,6 +74,7 @@ const OPTION_DESCRIPTION_KEYS = { 'analyze|--worker-timeout ': 'help.option.analyze.workerTimeout', 'analyze|--wal-checkpoint-threshold ': 'help.option.analyze.walCheckpointThreshold', 'analyze|--memory-budget ': 'help.option.analyze.memoryBudget', + 'analyze|--resource-budget ': 'help.option.analyze.resourceBudget', 'analyze|--workers ': 'help.option.analyze.workers', 'analyze|--max-processes ': 'help.option.analyze.maxProcesses', 'analyze|--max-process-branching ': 'help.option.analyze.maxProcessBranching', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 58105eccc..8f041070d 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -311,6 +311,8 @@ export const en = { 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).', 'help.option.analyze.memoryBudget': 'Main-thread V8 heap size in MB for analyze (integer >= 200). Re-runs analyze with exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size pin; parse workers keep their own heap caps.', + 'help.option.analyze.resourceBudget': + 'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.', 'help.option.analyze.workers': 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', 'help.option.analyze.maxProcesses': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index f06374781..f07717d4f 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -284,6 +284,8 @@ export const zhCN = { 'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值（字节，整数 >= -1；默认：67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。', 'help.option.analyze.memoryBudget': 'analyze 主线程 V8 堆大小（MB，整数 >= 200）。以该堆大小重新运行 analyze，覆盖按 RAM/cgroup 自动计算的上限及任何 --max-old-space-size 设置；解析 worker 各自保留独立的堆上限。', + 'help.option.analyze.resourceBudget': + '分析进程的资源总预算（MiB），包括原生数据库缓冲池、解析 worker、嵌入模型和监督进程。', 'help.option.analyze.workers': '解析 worker 池大小（>=1）。默认：cores-1，最多 16，按仓库规模自适应。', 'help.option.analyze.maxProcesses': diff --git a/gitnexus/src/core/analysis-embedding-phase.ts b/gitnexus/src/core/analysis-embedding-phase.ts index 4e4e2c99a..6dfe67396 100644 --- a/gitnexus/src/core/analysis-embedding-phase.ts +++ b/gitnexus/src/core/analysis-embedding-phase.ts @@ -22,6 +22,8 @@ import { runPublishedEmbeddingJob } from './embeddings/embedding-job.js'; import { drainEmbeddingSidecarForAnalysis } from './embeddings/embedding-sidecar-client.js'; import { deriveEmbeddingCap } from './embedding-mode.js'; import { embeddingProgressFromState, type EmbeddingJobProgress } from './analysis-progress.js'; +import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from './analyzer-identity.js'; +import { resolveGraphPath } from '../storage/shared-store.js'; export function canPromoteLegacyGraph(metadata: RepoMeta): boolean { return ( @@ -30,7 +32,7 @@ export function canPromoteLegacyGraph(metadata: RepoMeta): boolean { !metadata.graphWriteCollapsed && !metadata.embeddingCheckpoint?.recovery && metadata.schemaFingerprint === SCHEMA_FINGERPRINT && - metadata.runnerIdentity?.schemaVersion === 4 && + metadata.runnerIdentity?.schemaVersion === ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION && metadata.parseCacheIdentity !== undefined && metadata.fileHashes !== undefined ); @@ -215,7 +217,7 @@ export async function runAnalysisEmbeddingPhase( resources, signal, ); - const graphPath = path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'); + const graphPath = resolveGraphPath(metadataDirectory, metadata); let progress: EmbeddingJobProgress = { phase: 'preparing' }; callbacks.onProgress('graph-ready', 90, 'Graph available; preparing embeddings', { graphPhase: 'ready', diff --git a/gitnexus/src/core/analysis-graph-verification.ts b/gitnexus/src/core/analysis-graph-verification.ts index fdc471d6d..895b1ece6 100644 --- a/gitnexus/src/core/analysis-graph-verification.ts +++ b/gitnexus/src/core/analysis-graph-verification.ts @@ -6,6 +6,7 @@ import { closeLbugConnection, openLbugConnection } from './lbug/lbug-config.js'; import { assertLbugPublicationReady } from './lbug/publication.js'; import { getFtsIndexes } from './search/fts-schema.js'; import { verifySearchFTSIndexes } from './search/fts-indexes.js'; +import { ExtensionManager } from './lbug/extension-loader.js'; function count(value: unknown): number { if (typeof value !== 'number' && typeof value !== 'bigint') @@ -59,7 +60,12 @@ export async function verifyPublishedAnalysisGraph( ) throw new Error('Prepared graph counts differ after writer release; artifacts retained'); if (metadata.capabilities?.fts.status === 'available') { - await query('LOAD FTS'); + if ( + !(await new ExtensionManager().ensure(query, 'fts', 'publication verification', { + policy: 'load-only', + })) + ) + throw new Error('Prepared graph FTS extension could not be loaded; artifacts retained'); const missing = await verifySearchFTSIndexes(query, getFtsIndexes(metadata.ftsProfile)); if (missing.length) throw new Error(`Prepared graph search indexes are incomplete: ${missing.join(', ')}`); diff --git a/gitnexus/src/core/analysis-host-resources.ts b/gitnexus/src/core/analysis-host-resources.ts index 18cf6e924..30fba8255 100644 --- a/gitnexus/src/core/analysis-host-resources.ts +++ b/gitnexus/src/core/analysis-host-resources.ts @@ -41,15 +41,20 @@ export function resolveHostAnalysisResources( if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error('Analysis concurrency must be a positive safe integer'); const http = Boolean(process.env.GITNEXUS_EMBEDDING_URL && process.env.GITNEXUS_EMBEDDING_MODEL); + const effectiveMemoryBytes = Math.floor(effectiveRamBytes() / concurrency); const request: AnalysisResourceRequest = { - effectiveMemoryBytes: Math.floor(effectiveRamBytes() / concurrency), + effectiveMemoryBytes, availableCpus: Math.max(1, Math.floor(os.availableParallelism() / concurrency)), - totalMb: options.totalMb, + totalMb: + options.totalMb ?? + (concurrency > 1 ? Math.floor((effectiveMemoryBytes / MIB) * 0.75) : undefined), mainHeapMb: options.mainHeapMb, allowHeapOvercommit: options.mainHeapMb !== undefined && options.totalMb === undefined, workers: options.workers ?? positiveEnvironment('GITNEXUS_WORKER_POOL_SIZE'), workerHeapMb: positiveEnvironment('GITNEXUS_WORKER_HEAP_MB'), - embeddingThreads: positiveEnvironment('GITNEXUS_EMBEDDING_THREADS'), + embeddingThreads: options.includeEmbeddings + ? positiveEnvironment('GITNEXUS_EMBEDDING_THREADS') + : undefined, preferredWorkers: 16, includeParsing: options.includeParsing, graphPoolBytes: positiveEnvironment('GITNEXUS_LBUG_BUFFER_POOL_SIZE'), diff --git a/gitnexus/src/core/analysis-phase-supervisor.ts b/gitnexus/src/core/analysis-phase-supervisor.ts index 4f4d66a4c..14f6aafbf 100644 --- a/gitnexus/src/core/analysis-phase-supervisor.ts +++ b/gitnexus/src/core/analysis-phase-supervisor.ts @@ -1,6 +1,5 @@ /** Filesystem-only handoff from an exit-only graph writer. No model/native imports. */ import path from 'node:path'; -import type { ChildProcess } from 'node:child_process'; import type { RepoMeta } from '../storage/repo-meta.js'; import { requireExclusiveIndexLock } from '../storage/index-lock.js'; import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; @@ -54,38 +53,6 @@ export function projectPreparedAnalysisHandoff(value: unknown): PreparedAnalysis return projected; } -/** Call when the process is spawned. A prepared/complete IPC is never release - * evidence. Errors reject without authorizing any continuation or killing the - * child; its existing launcher retains cancellation/abandonment ownership. */ -export function waitForAnalysisProcessExit(child: ChildProcess): Promise { - return new Promise((resolve, reject) => { - const cleanup = () => { - child.removeListener('exit', onExit); - child.removeListener('error', onError); - }; - const onExit = (code: number | null, signal: NodeJS.Signals | null) => { - cleanup(); - if (code === 0 && signal === null) resolve(); - else - reject( - new Error( - `Graph process exited unsuccessfully (${signal ?? code ?? 'unknown'}); prepared artifacts retained`, - ), - ); - }; - const onError = (error: Error) => { - cleanup(); - reject(error); - }; - if (child.exitCode !== null || child.signalCode !== null) { - onExit(child.exitCode, child.signalCode); - return; - } - child.once('exit', onExit); - child.once('error', onError); - }); -} - export interface AnalysisPhasePublicationActions extends AnalysisPublicationActions { /** Re-resolve checkout placement, current HEAD and the registered name. */ resolveTarget: () => Promise; diff --git a/gitnexus/src/core/analysis-progress.ts b/gitnexus/src/core/analysis-progress.ts index 5830024e3..f56f5021a 100644 --- a/gitnexus/src/core/analysis-progress.ts +++ b/gitnexus/src/core/analysis-progress.ts @@ -17,6 +17,7 @@ export interface EmbeddingJobProgress { reusedNodes?: number; pendingNodes?: number; failedNodes?: number; + excludedNodes?: number; chunks?: number; } @@ -51,6 +52,7 @@ const publicEmbeddingProgress = ( 'reusedNodes', 'pendingNodes', 'failedNodes', + 'excludedNodes', 'chunks', ] as const) { const value = progress[key]; @@ -99,12 +101,17 @@ export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJ state.completedNodes, state.reusedNodes, state.failedNodes, + state.excludedNodes ?? 0, state.chunks, ]; const pendingUnits = state.totalUnits - state.completedUnits - state.reusedUnits - state.failedUnits; const pendingNodes = - state.totalNodes - state.completedNodes - state.reusedNodes - state.failedNodes; + state.totalNodes - + state.completedNodes - + state.reusedNodes - + state.failedNodes - + (state.excludedNodes ?? 0); const activeUnit = state.active?.start?.unit ?? null; if ( counts.some((n) => !Number.isSafeInteger(n) || n < 0) || @@ -143,6 +150,7 @@ export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJ reusedNodes: state.reusedNodes, pendingNodes, failedNodes: state.failedNodes, + ...(state.excludedNodes ? { excludedNodes: state.excludedNodes } : {}), chunks: state.chunks, }; } diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts index 25492c073..908f2e61c 100644 --- a/gitnexus/src/core/analysis-publication.ts +++ b/gitnexus/src/core/analysis-publication.ts @@ -8,6 +8,8 @@ import type { RepoMeta } from '../storage/repo-meta.js'; import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from '../storage/storage-constants.js'; import { preserveVectorCoverage } from '../storage/vector-generation.js'; import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; +import { readProcStartTime } from '../storage/index-lock.js'; +import { reclaimAnalysisArtifacts } from '../storage/analysis-artifact-cleanup.js'; const RECEIPT = 'analysis-publication.json'; const GENERATION = /^[a-f0-9]{32}$/; @@ -17,6 +19,10 @@ const STAGING = /^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; const MAX_RECEIPT_BYTES = 4096; const MAX_METADATA_BYTES = 256 * 1024 * 1024; const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); +// Share the incarnation across loaders of this module within one process. +const INSTANCE_KEY = Symbol.for('gitnexus.analysis-publication.writer-instance'); +const processIdentity = process as unknown as Record; +const WRITER_INSTANCE = (processIdentity[INSTANCE_KEY] ??= randomUUID().replaceAll('-', '')); export interface PreparedAnalysisPublication { version: 2; @@ -34,6 +40,8 @@ export interface PreparedAnalysisPublication { redirectedMetadataDigest?: string; /** Exit-only callers may not publish while this process is still alive. */ writerPid?: number; + writerStartTime?: string; + writerInstance?: string; /** A hard-link destination for the previous legacy graph; never copied. */ legacyGraphFile?: string; } @@ -71,6 +79,8 @@ function validateReceipt(value: unknown): asserts value is PreparedAnalysisPubli (r.redirectedMetadataDigest !== undefined && (!r.legacyGraphFile || !DIGEST.test(r.redirectedMetadataDigest))) || (r.writerPid !== undefined && (!Number.isSafeInteger(r.writerPid) || r.writerPid < 1)) || + (r.writerStartTime !== undefined && (!r.writerPid || !/^\d{1,32}$/.test(r.writerStartTime))) || + (r.writerInstance !== undefined && (!r.writerPid || !GENERATION.test(r.writerInstance))) || (r.legacyGraphFile !== undefined && (r.mode !== 'staged' || !PRIVATE_GRAPH.test(r.legacyGraphFile) || @@ -96,11 +106,14 @@ async function regularFile(file: string): Promise { } async function boundedRead(file: string, maximum: number): Promise { - if (!(await regularFile(file))) - throw Object.assign(new Error('Publication artifact missing'), { code: 'ENOENT' }); + const before = await fs.lstat(file); + if (!before.isFile() || before.isSymbolicLink()) + throw new Error('Publication artifact must be an owned regular file'); const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); try { const stat = await handle.stat(); + if (stat.dev !== before.dev || stat.ino !== before.ino) + throw new Error('Publication artifact changed while opening'); if (!stat.isFile() || stat.size > maximum) throw new Error('Publication artifact exceeds its receipt bound'); const buffer = Buffer.alloc(stat.size + 1); @@ -217,7 +230,7 @@ export async function prepareAnalysisPublication( const mode = options.mode ?? 'staged'; const sourceFile = path.basename(buildPath); if ( - path.dirname(path.resolve(buildPath)) !== root || + (await fs.realpath(path.dirname(buildPath))) !== root || (mode === 'staged' ? !STAGING.test(sourceFile) : sourceFile !== 'lbug' && !PRIVATE_GRAPH.test(sourceFile)) @@ -260,7 +273,13 @@ export async function prepareAnalysisPublication( metadataBytes: bytes.length, metadataDigest: hash(bytes), priorMetadataDigest: metadataDigest(prior), - ...(options.waitForProcessExit ? { writerPid: process.pid } : {}), + ...(options.waitForProcessExit + ? { + writerPid: process.pid, + writerInstance: WRITER_INSTANCE, + writerStartTime: readProcStartTime(process.pid) ?? undefined, + } + : {}), ...(mode === 'staged' && (await regularFile(path.join(root, 'lbug'))) ? { legacyGraphFile: `lbug.generation.${randomUUID().replaceAll('-', '')}` } : {}), @@ -363,7 +382,16 @@ async function retireLegacyGraph( await actions.saveMetadata(redirected); await syncPublishedMetadata(root, redirected); } - await fs.unlink(legacy); + try { + await fs.unlink(legacy); + } catch (error) { + if (['EBUSY', 'EPERM', 'EACCES'].includes((error as NodeJS.ErrnoException).code ?? '')) + throw new Error( + 'Cannot retire the legacy graph while a reader holds it. Stop older MCP/serve processes and rerun analyze; the prepared graph and embeddings are retained.', + { cause: error }, + ); + throw error; + } await syncEmbeddingJobDirectory(root); } @@ -380,11 +408,31 @@ export async function publishPreparedAnalysis( const receipt = await readPreparedAnalysis(root); if (!receipt) return undefined; if (receipt.writerPid !== undefined) { - try { - process.kill(receipt.writerPid, 0); - throw new Error('Graph writer has not exited; publication must wait for physical release'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + const deadline = performance.now() + 2000; + for (;;) { + // A live, recycled PID is not the process that prepared this receipt. + if ( + receipt.writerPid === process.pid && + receipt.writerInstance && + receipt.writerInstance !== WRITER_INSTANCE + ) + break; + const started = readProcStartTime(receipt.writerPid); + if (receipt.writerStartTime && started && receipt.writerStartTime !== started) break; + try { + process.kill(receipt.writerPid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') break; + throw new Error( + `Cannot verify graph writer PID ${receipt.writerPid}: ${error instanceof Error ? error.message : String(error)}. Run analyze as the original index owner after that process exits; artifacts retained.`, + { cause: error }, + ); + } + if (performance.now() >= deadline) + throw new Error( + `Graph writer PID ${receipt.writerPid} has not exited; publication must wait for physical release. Stop that analysis process and retry; artifacts retained.`, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); } } const bytes = await boundedRead(path.join(root, receipt.metadataFile), receipt.metadataBytes); @@ -431,5 +479,6 @@ export async function publishPreparedAnalysis( await syncEmbeddingJobDirectory(root); await fs.unlink(path.join(root, receipt.metadataFile)); await syncEmbeddingJobDirectory(root); + await reclaimAnalysisArtifacts(root, metadata, previous?.graphPath ?? receipt.legacyGraphFile); return metadata; } diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index c5eee9f93..71acc3342 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -1,3 +1,4 @@ +import { heapCapMbFor } from './ingestion/utils/effective-ram.js'; /** Pure admission policy. Reservations are not an OS-enforced RSS limit. */ const MIB = 1024 * 1024; const GRAPH_POOL_MB = 256; @@ -86,6 +87,7 @@ export function resolveAnalysisResourceBudget( : integer(request.embeddingThreads, 'embedding threads'); if (embeddingThreads > cpus) throw new Error('Embedding threads exceed the CPU allowance'); const availableMb = Math.floor(memory / MIB); + const automatic = request.totalMb === undefined; let totalMb = request.totalMb === undefined ? Math.floor(availableMb * 0.75) @@ -105,6 +107,7 @@ export function resolveAnalysisResourceBudget( : integer(request.graphPoolBytes, 'graph pool bytes', 8 * MIB); // Zero is the native engine's unbounded override, never a bounded allocation. const graphPoolMb = Math.ceil(graphPoolBytes / MIB); + const vectorPoolMb = Math.max(VECTOR_POOL_MB, Math.min(4096, Math.floor(totalMb / 8))); const modelMb = integer(request.modelMemoryMb, 'model reservation', 0); if (modelMb > 0 && modelMb < 512) throw new Error( @@ -114,8 +117,20 @@ export function resolveAnalysisResourceBudget( // explicit budgets must be able to admit it without truncating its content. const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32))); const embeddingFixedMb = - headroomMb + supervisorMb + graphPoolMb + VECTOR_POOL_MB + modelMb + queueMb; + headroomMb + supervisorMb + graphPoolMb + vectorPoolMb + modelMb + queueMb; const includeParsing = request.includeParsing !== false; + const cpuWorkers = Math.max(1, cpus - 1); + const desiredWorkers = + request.workers === undefined + ? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers) + : integer(request.workers, 'parser workers'); + const explicitWorkerHeapMb = + request.workerHeapMb === undefined + ? undefined + : integer(request.workerHeapMb, 'worker heap reservation'); + const automaticWorkerHeapMb = + explicitWorkerHeapMb ?? + Math.min(4096, Math.max(MIN_WORKER_HEAP_MB, Math.floor(availableMb / 2 / desiredWorkers))); const parseFixedMb = headroomMb + supervisorMb + @@ -124,18 +139,33 @@ export function resolveAnalysisResourceBudget( let maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); const mainHeapMb = request.mainHeapMb === undefined - ? Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) + ? automatic + ? heapCapMbFor(request.effectiveMemoryBytes) + : Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) : integer(request.mainHeapMb, 'main heap reservation'); - // --memory-budget predates the family budget and deliberately permits a heap - // above RAM (with the CLI's existing swap warning). Extend only the finite - // heap reservation; do not pretend the host has more RAM or enlarge native - // pools/queues. An explicit --resource-budget always remains authoritative. - if ( - request.allowHeapOvercommit && - request.totalMb === undefined && - request.mainHeapMb !== undefined - ) { - totalMb += Math.max(0, mainHeapMb - maximumMainMb); + // Automatic sizing preserves the established independent V8 ceilings: the existing RAM-aware + // main heap, half RAM shared by parser heaps. Their finite caps + // can sum above RAM; they are not simultaneous resident allocations. Only an + // explicit --resource-budget imposes a sum ceiling. A heap override must not + // silently consume worker capacity or inflate the native pools/queues. + if (automatic) { + if ( + mainHeapMb > availableMb && + !request.allowHeapOvercommit && + request.mainHeapMb !== undefined + ) + throw new Error( + 'Resource budget cannot admit a main heap above effective memory without an explicit heap override', + ); + totalMb = Math.max( + totalMb, + mainHeapMb + embeddingFixedMb, + headroomMb + + supervisorMb + + graphPoolMb + + mainHeapMb + + (includeParsing ? desiredWorkers * (automaticWorkerHeapMb + WORKER_NATIVE_MB) : 0), + ); maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); } const totalBytes = bytes(totalMb, 'resource budget'); @@ -150,19 +180,10 @@ export function resolveAnalysisResourceBudget( ); } - const cpuWorkers = Math.max(1, cpus - 1); - const desiredWorkers = - request.workers === undefined - ? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers) - : integer(request.workers, 'parser workers'); // Explicit worker counts intentionally permit CPU oversubscription, as the // parser's existing override does. Reserve every worker's heap/native memory // below; automatic concurrency remains bounded by the usable CPU count. const workerBudgetMb = totalMb - headroomMb - supervisorMb - graphPoolMb - mainHeapMb; - const explicitWorkerHeapMb = - request.workerHeapMb === undefined - ? undefined - : integer(request.workerHeapMb, 'worker heap reservation'); if ( includeParsing && explicitWorkerHeapMb !== undefined && @@ -172,18 +193,22 @@ export function resolveAnalysisResourceBudget( } const workers = !includeParsing ? 0 - : request.workers === undefined - ? Math.min( - desiredWorkers, - Math.floor( - workerBudgetMb / ((explicitWorkerHeapMb ?? MIN_WORKER_HEAP_MB) + WORKER_NATIVE_MB), - ), - ) - : desiredWorkers; + : automatic + ? desiredWorkers + : request.workers === undefined + ? Math.min( + desiredWorkers, + Math.floor( + workerBudgetMb / ((explicitWorkerHeapMb ?? MIN_WORKER_HEAP_MB) + WORKER_NATIVE_MB), + ), + ) + : desiredWorkers; const workerHeapMb = !includeParsing ? 0 - : (explicitWorkerHeapMb ?? - Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB)); + : automatic + ? automaticWorkerHeapMb + : (explicitWorkerHeapMb ?? + Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB)); const parseReservedMb = headroomMb + supervisorMb + @@ -201,7 +226,7 @@ export function resolveAnalysisResourceBudget( const embeddingReservedMb = mainHeapMb + embeddingFixedMb; return Object.freeze({ totalBytes, - automatic: request.totalMb === undefined, + automatic, headroomBytes: bytes(headroomMb, 'headroom'), supervisorBytes: bytes(supervisorMb, 'supervisor reservation'), mainHeapMb, @@ -215,7 +240,7 @@ export function resolveAnalysisResourceBudget( }), embedding: Object.freeze({ graphPoolBytes, - vectorPoolBytes: bytes(VECTOR_POOL_MB, 'vector pool'), + vectorPoolBytes: bytes(vectorPoolMb, 'vector pool'), modelBytes: bytes(modelMb, 'model reservation'), modelHeapMb: modelMb === 0 ? 0 : Math.min(1024, Math.floor(modelMb / 4)), queueBytes: bytes(queueMb, 'embedding queues'), diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts index 5bef87ebb..6dfba2a9c 100644 --- a/gitnexus/src/core/embeddings/chunker.ts +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -9,7 +9,7 @@ export { type Chunk, characterChunk } from './character-chunk.js'; -import { characterChunk, iterateCharacterChunks } from './character-chunk.js'; +import { iterateCharacterChunks } from './character-chunk.js'; import type { Chunk } from './character-chunk.js'; import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js'; import { buildLineIndex, resolveChunkLines } from './line-index.js'; @@ -693,7 +693,7 @@ const projectClassChunks = ( overlap: number, strategy: EmbeddingConfig['chunkStrategy'] = 'ast', expectedName?: string, -): EmbeddingChunk[] => { +): Iterable => { const target = tree && findDeclarationNode(tree.rootNode, expectedName); const body = target && getDeclarationBodyNode(target); if (!body || target.hasError) return [classMetadataChunk(startLine, endLine)]; @@ -784,7 +784,7 @@ const projectClassChunks = ( const projected = pieces.join(''); const chunks = strategy === 'fixed' - ? characterChunk(projected, 1, 1, chunkSize, overlap) + ? iterateCharacterChunks(projected, 1, 1, chunkSize, overlap) : chunkByUnits( projected, buildLineIndex(projected), @@ -811,17 +811,19 @@ const projectClassChunks = ( segment.source + Math.max(0, Math.min(offset - segment.start, segment.end - segment.start)) ); }; - return Array.from(chunks, (chunk) => { - const startOffset = sourceOffset(chunk.startOffset); - const endOffset = sourceOffset(chunk.endOffset - 1) + 1; - return { - ...chunk, - startOffset, - endOffset, - ...resolveChunkLines(originalLines, startOffset, endOffset, startLine), - classDeclaration: header, - }; - }); + return (function* () { + for (const chunk of chunks) { + const startOffset = sourceOffset(chunk.startOffset); + const endOffset = sourceOffset(chunk.endOffset - 1) + 1; + yield { + ...chunk, + startOffset, + endOffset, + ...resolveChunkLines(originalLines, startOffset, endOffset, startLine), + classDeclaration: header, + }; + } + })(); }; const collectDeclarationUnits = ( diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index 7fc48e629..e705a075d 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -99,10 +99,15 @@ export async function runEmbeddingJob( intent: EmbeddingUnitIntent, ): Promise => { signal?.throwIfAborted(); + const history = journal.snapshot().active?.inputFailures ?? intent.inputFailures ?? {}; + if (Object.keys(history).some((id) => !unit.nodes.some((node) => node.id === id))) + throw new Error('Embedding rejection history does not match its manifest unit'); + const excluded = new Set(Object.keys(history).filter((id) => history[id] >= 3)); const complete = options.revalidateStoredInputs ? new Map() : await store.completedNodes(unit.nodes.map((node) => node.id)); - const missing = unit.nodes.filter((node) => !complete.has(node.id)); + for (const id of excluded) complete.delete(id); + const missing = unit.nodes.filter((node) => !complete.has(node.id) && !excluded.has(node.id)); const retainedBytes = missing.reduce( (n, key) => n + 2 * (key.contentBytes + key.metadataBytes), 0, @@ -132,9 +137,10 @@ export async function runEmbeddingJob( `Embedding unit ${unit.ordinal} requires a larger resource budget; no source text was truncated`, ); if (!journal.snapshot().active) await journal.begin(intent); + if (excluded.size) await store.excludeNodes([...excluded]); progress(); if (!missing.length) { - await store.checkpointUnit([...complete.values()]); + if (complete.size) await store.checkpointUnit([...complete.values()]); await journal.acknowledge({ reusedNodes: complete.size, chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), @@ -158,6 +164,7 @@ export async function runEmbeddingJob( { contentHash: string; chunks: number; ended: boolean; inputsDigest?: string } >(); const failed = new Set(); + const rejectedInputs = new Set(); let failureCause: 'input' | 'provider' = 'input'; let abort: unknown; let reusedNodes = complete.size; @@ -196,6 +203,7 @@ export async function runEmbeddingJob( if ('error' in result) { for (const row of rows) failed.add(row.nodeId); if (isInputRejection(result.error)) { + for (const row of rows) rejectedInputs.add(row.nodeId); for (const row of rows) if (rejected.size < 5) rejected.add(row.nodeId); } else { failureCause = 'provider'; @@ -458,6 +466,14 @@ export async function runEmbeddingJob( } if (copyBatch.length) await flushCopies(); if (!abort) await flush(); + const attempts = await journal.recordInputRejections([...rejectedInputs]); + for (const id of rejectedInputs) { + if (attempts[id] < 3) continue; + await store.excludeNodes([id]); + excluded.add(id); + generated.delete(id); + failed.delete(id); + } const finished = [ ...[...generated] .filter(([id, node]) => node.ended && !failed.has(id)) @@ -477,7 +493,7 @@ export async function runEmbeddingJob( await store.markSelected(journal.snapshot().identity, [...complete.keys()]); await store.checkpointUnit([...complete.values()]); } - if (complete.size === unit.nodes.length) { + if (complete.size + excluded.size === unit.nodes.length) { await journal.acknowledge({ reusedNodes, chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), @@ -531,9 +547,15 @@ export async function runEmbeddingJob( } signal?.throwIfAborted(); if (options.revalidateStoredInputs && !store.isSealed) - await store.pruneUnselected(journal.snapshot().identity, manifest.totalNodes); + await store.pruneUnselected( + journal.snapshot().identity, + manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0), + ); signal?.throwIfAborted(); - const indexKind = await store.seal(manifest.totalNodes, journal.snapshot().chunks); + const indexKind = await store.seal( + manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0), + journal.snapshot().chunks, + ); return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; } finally { sourceReader = undefined; diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 55fc210fc..35a90811e 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -544,12 +544,22 @@ export async function runPublishedEmbeddingJob( dimensions: vector.dimensions, nodes: manifest.totalNodes, chunks: result.state.chunks, + ...(result.state.excludedNodes + ? { + excludedNodes: result.state.excludedNodes, + exclusionReason: 'input-rejected' as const, + } + : {}), indexKind: result.indexKind, }, storeResources, options.metadata, ); if (!published) return { metadata, alreadyComplete: false, phase: 'indexing', ...result }; + if (result.state.excludedNodes) + options.log?.( + `Skipped ${result.state.excludedNodes} node(s) after three deterministic embedding input rejections; compatible vectors were published.`, + ); if (journal.snapshot().phase !== 'ready') await journal.ready(); options.onProgress?.(journal.snapshot(), result.peakQueueBytes); return { diff --git a/gitnexus/src/core/embeddings/embedding-sidecar.ts b/gitnexus/src/core/embeddings/embedding-sidecar.ts index 70f3aa936..93f0de9c8 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar.ts @@ -16,6 +16,10 @@ process.on('message', (msg: SidecarRequest) => { void handle(msg); }); +// This is a standalone process, not a worker thread. If its owner exits or is +// killed after cancellation grace, release the model instead of orphaning it. +process.once('disconnect', () => process.exit(1)); + async function handle(msg: SidecarRequest): Promise { try { switch (msg.type) { diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 946881361..7ed012028 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -50,7 +50,13 @@ export interface EmbeddingRequestOptions { maxResponseBytes?: number; } -class EmbeddingResponseBudgetError extends Error {} +// A local resource limit aborts the request without retrying or penalising +// endpoint health. resilientFetch treats AbortError DOMExceptions as neutral. +class EmbeddingResponseBudgetError extends DOMException { + constructor(message: string) { + super(message, 'AbortError'); + } +} /** Bound encoded bytes before JSON decoding; Content-Length alone is untrusted. */ async function readEmbeddingResponse(response: Response, maximum?: number): Promise { @@ -537,7 +543,7 @@ const httpEmbedBatch = async ( }; } catch (err) { if (err instanceof EmbeddingResponseBudgetError) - throw new RetryableEmbeddingBodyError( + throw new EmbeddingResponseBudgetError( `Embedding endpoint exceeded its ${requestOptions.maxResponseBytes}-byte response budget (${safeUrl(url)}, batch ${batchIndex})`, ); // Not every `.json()` rejection is a parse error: the per-attempt @@ -588,6 +594,9 @@ const httpEmbedBatch = async ( }, ); } catch (err) { + if (err instanceof EmbeddingResponseBudgetError) { + throw new HttpEmbeddingError(err.message, { cause: err }); + } if ( requestOptions.signal?.aborted || (err instanceof DOMException && err.name === 'AbortError') diff --git a/gitnexus/src/core/embeddings/vector-generation-search.ts b/gitnexus/src/core/embeddings/vector-generation-search.ts index 6f028a0b7..2b1b25ef8 100644 --- a/gitnexus/src/core/embeddings/vector-generation-search.ts +++ b/gitnexus/src/core/embeddings/vector-generation-search.ts @@ -16,6 +16,7 @@ import { getExactScanLimit } from '../platform/capabilities.js'; import { collectBestChunks } from './types.js'; import { rankExactEmbeddingRows, type ExactSearchChunk } from './exact-search.js'; import { assertQueryIdentity, rethrowSymbolIdentityError } from '../lbug/query-result-integrity.js'; +import { logger } from '../logger.js'; export interface VectorSearchSnapshot { metadataDirectory: string; @@ -75,7 +76,13 @@ export async function searchVectorGeneration( throw new Error('Published vector identity is unverified'); const annReady = selected.generation.indexKind === 'hnsw' && - (await ensureVectorExtension(dbPath).catch(() => false)); + (await ensureVectorExtension(dbPath).catch((error: unknown) => { + logger.warn( + { err: error }, + 'Published VECTOR extension could not load; checking the bounded exact-search fallback', + ); + return false; + })); if (!annReady && selected.generation.chunks > exactLimit) return []; const identity = await resolveIdentity(); const vectorIdentity = createHash('sha256') @@ -103,7 +110,7 @@ export async function searchVectorGeneration( return row as ExactSearchChunk; }); }); - return [...best].map(([nodeId, chunk]) => ({ nodeId, ...chunk })); + if (best.size > 0) return [...best].map(([nodeId, chunk]) => ({ nodeId, ...chunk })); } catch (error) { rethrowSymbolIdentityError(error); if (selected.generation.chunks > exactLimit) throw error; diff --git a/gitnexus/src/core/embeddings/vector-publication.ts b/gitnexus/src/core/embeddings/vector-publication.ts index ab2e53e96..786a9859b 100644 --- a/gitnexus/src/core/embeddings/vector-publication.ts +++ b/gitnexus/src/core/embeddings/vector-publication.ts @@ -8,6 +8,7 @@ import { import { VectorStore } from '../lbug/vector-store.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { getExactScanLimit } from '../platform/capabilities.js'; +import { reclaimAnalysisArtifacts } from '../../storage/analysis-artifact-cleanup.js'; export interface VectorPublicationActions { loadMetadata: () => Promise; @@ -55,6 +56,7 @@ export async function publishVectorGeneration( : 'exact-scan' : 'unavailable', exactScanLimit, + ...(generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}), }, }, }; @@ -79,12 +81,16 @@ export async function publishVectorGeneration( ...resources, }); try { - if ((await store.seal(generation.nodes, generation.chunks)) !== generation.indexKind) + if ( + (await store.seal(generation.nodes - (generation.excludedNodes ?? 0), generation.chunks)) !== + generation.indexKind + ) throw new Error('Published vector index does not match the native artifact'); } finally { await store.close(); } await actions.saveMetadata(metadata); await syncPublishedMetadata(metadataDirectory, metadata); + await reclaimAnalysisArtifacts(metadataDirectory, metadata); return metadata; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index fb872edb5..818d08aa8 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -833,6 +833,7 @@ export async function runChunkedParseAndResolve( let parseCacheHitFileCount = 0; let reparsedFileCount = 0; let parseFailure: unknown; + let parsingFailed = false; try { // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch @@ -1243,6 +1244,7 @@ export async function runChunkedParseAndResolve( } let missIdx = 0; for (const entry of round.entries) { + options?.signal?.throwIfAborted(); if (entry.kind === 'hit') { const chunkWorkerData = mergeChunkResults( graph, @@ -1273,6 +1275,7 @@ export async function runChunkedParseAndResolve( * park it resolved for the next close to merge. */ const closeRound = async (): Promise => { + options?.signal?.throwIfAborted(); const started = await startRound(roundEntries); roundEntries = []; roundBudget.reset(); @@ -1307,6 +1310,7 @@ export async function runChunkedParseAndResolve( }; for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + options?.signal?.throwIfAborted(); if (heapProbeEveryN > 0 && chunkIdx > 0 && chunkIdx % heapProbeEveryN === 0) { logHeapProbe( `parse-chunk-${chunkIdx}`, @@ -1543,15 +1547,16 @@ export async function runChunkedParseAndResolve( } } catch (error) { parseFailure = error; + parsingFailed = true; throw error; } finally { try { await workerPool?.terminate(); } catch (drainError) { - if (parseFailure !== undefined) { + if (parsingFailed) { throw new AggregateError( [parseFailure, drainError], - 'Parsing failed and native workers have not drained', + `Parsing failed (${parseFailure instanceof Error ? parseFailure.message : String(parseFailure)}) and native workers have not drained`, { cause: parseFailure }, ); } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts index da8e4dd8f..79e4da9f5 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts @@ -203,6 +203,7 @@ export async function runPipeline( const results = new Map>(seed); for (const phase of sorted) { + ctx.options?.signal?.throwIfAborted(); const start = Date.now(); if (isDev) { diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 64b7a4522..486c67c9a 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -56,6 +56,8 @@ import { } from './pipeline-phases/index.js'; export interface PipelineOptions { + /** Cooperative cancellation checked between parser dispatch and graph phases. */ + signal?: AbortSignal; /** * Skip MRO, community detection, and process extraction for faster test runs. * The `pruneLocalSymbols` phase still runs — it is graph construction (it cleans @@ -390,6 +392,7 @@ export const runPipelineFromRepo = async ( onProgress: (progress: PipelineProgress) => void, options?: PipelineOptions, ): Promise => { + options?.signal?.throwIfAborted(); const graph = createKnowledgeGraph(); const pipelineStart = Date.now(); @@ -429,6 +432,7 @@ export const runPipelineFromRepo = async ( let results; try { results = await runPipeline(phases, ctx); + options?.signal?.throwIfAborted(); graphEmitManifest = graphEmitSink?.finalize(); } finally { // Release per-pair fds when the pipeline threw before finalize ran. diff --git a/gitnexus/src/core/ingestion/utils/effective-ram.ts b/gitnexus/src/core/ingestion/utils/effective-ram.ts index b5ea977c6..78aacc1ec 100644 --- a/gitnexus/src/core/ingestion/utils/effective-ram.ts +++ b/gitnexus/src/core/ingestion/utils/effective-ram.ts @@ -99,7 +99,7 @@ export function heapPressureRemedy( if (heapLimitBytes < autoCapBytes * 0.9) { if (source === 'budget') { return ( - `This machine has more memory available: raise --memory-budget, or omit it ` + + `This machine has more memory available: raise --memory-budget (and --resource-budget if set), or omit the limits ` + `so gitnexus sizes its heap to the machine automatically.` ); } diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index 360da913b..1b521eed4 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -1145,7 +1145,7 @@ export const createWorkerPool = ( safePoint: Promise; }; const retiredWorkers = new Set(); - const pendingWorkerReleases = new Set>(); + const pendingWorkerReleases = new Map, number>(); const respawnCount: number[] = new Array(size).fill(0); const activeSlots: Set = new Set(); // Layer 3 (quarantine): tracked via the dedicated `quarantine.ts` @@ -1607,7 +1607,7 @@ export const createWorkerPool = ( completeRelease(); }; if (requireDrain) { - pendingWorkerReleases.add(termination); + pendingWorkerReleases.set(termination, workerIndex); // A rejected terminate request is not proof of release. An eventual // physical exit is, even when the termination promise rejected. existing.once('exit', released); @@ -1645,7 +1645,7 @@ export const createWorkerPool = ( return false; if (retiredWorkers.has(record)) return false; } - if (pendingWorkerReleases.size > 0 || stopped) return false; + if ([...pendingWorkerReleases.values()].includes(workerIndex) || stopped) return false; } const replacement = spawnAndCapture(workerUrl); try { @@ -2466,7 +2466,9 @@ export const createWorkerPool = ( activeSlots.clear(); if (requireDrain && (retiredWorkers.size > 0 || pendingWorkerReleases.size > 0)) throw new Error( - 'Parser workers have not released their native resource reservation; refusing to start another analysis phase. Workers will exit at their next safe point.', + 'Parser workers have not released their native resource reservation; refusing to start another analysis phase. ' + + `Suspect files: ${quarantine.snapshot().slice(0, 5).join(', ') || 'see the last parser progress message'}. ` + + 'Workers will exit at their next safe point. Exclude a repeatedly stuck file with .gitnexusignore and retry; do not kill a native worker thread.', ); }; diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 319d7e574..3d1c894af 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -12,6 +12,7 @@ import { assertValidGraphNodeId } from '../graph/node-identity.js'; import { embeddingJobDirectory } from '../../storage/embedding-job.js'; import { getExactScanLimit } from '../platform/capabilities.js'; import type { EmbeddingReuseOptions } from '../embeddings/embedding-inputs.js'; +import { logger } from '../logger.js'; export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v2'; const BLOCK_ROWS = 256; @@ -169,6 +170,20 @@ export class VectorStore { store.sealed = rows[0].complete === true; if (options.mode === 'read' && !store.sealed) throw new Error('Vector artifact is not complete'); + if (options.mode === 'resume' && !store.sealed) { + const indexes = await store.query('CALL SHOW_INDEXES() RETURN *'); + if (indexes.some((index) => index.index_name === 'code_embedding_idx')) { + const manager = new ExtensionManager(); + if ( + !(await manager.ensure((sql) => store.query(sql), 'vector', 'vector resume', { + policy: 'load-only', + })) + ) + throw new Error( + 'Cannot load VECTOR for the retained index; artifacts preserved. Restore the extension before retrying.', + ); + } + } } return store; } catch (error) { @@ -404,6 +419,23 @@ export class VectorStore { return this.beginNodes([{ nodeId, contentHash }]); } + /** The durable journal owns the three-attempt exclusion proof. Remove only + * those rejected nodes' partial rows and checkpoint before acknowledging. */ + async excludeNodes(ids: readonly string[]): Promise { + await this.beginNodes(ids.map((nodeId) => ({ nodeId, contentHash: 'input-rejected' }))); + await this.exclusive(async () => { + await this.query('UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) DELETE n', { + ids: [...ids], + }); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, true); + } + async beginNodes(nodes: readonly VectorNodeInput[]): Promise { if ( !nodes.length || @@ -906,19 +938,35 @@ export class VectorStore { // A complete artifact may be searched exactly if optional ANN fails. // Native corruption/durability errors still surface at checkpoint. if (/already exists/i.test(String(error))) kind = 'hnsw'; + else + logger.warn( + { err: error, chunks: totalChunks, bufferPoolBytes: this.options.bufferPoolBytes }, + 'Vector index build failed; completed vectors retained', + ); } } if (kind === 'hnsw') { try { await this.verifyVectorIndex(); - } catch { + } catch (error) { + logger.warn( + { err: error }, + 'Vector index verification failed; dropping the unpublished index for retry', + ); + await this.query("CALL DROP_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx')"); kind = 'exact'; } } // Large exact scans are deliberately unavailable to readers. Keep this // complete work unpublished and writable so a later run can retry ANN // without a single new provider request. - if (kind === 'exact' && totalChunks > getExactScanLimit()) return kind; + if (kind === 'exact' && totalChunks > getExactScanLimit()) { + logger.warn( + { chunks: totalChunks, bufferPoolBytes: this.options.bufferPoolBytes }, + 'Vector index is pending beyond the exact-scan limit. Increase --resource-budget and retry; completed vectors will be reused.', + ); + return kind; + } await this.query( "MATCH (a:EmbeddingArtifact {id:'artifact'}) SET a.complete=true, a.indexKind=$kind", { kind }, diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index c99b681a5..6b3c3085c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -11,7 +11,7 @@ import { detectGraphWriteCollapse, - getIndexIncompleteReasons, + getGraphIncompleteReasons, type GraphWriteCollapseVerdict, } from './index-freshness.js'; import { @@ -1585,6 +1585,7 @@ export async function continuePreparedAnalysis( }; }; const finalize = async (metadata: RepoMeta): Promise => { + const prior = options.noShare ? await optedInSlotToLeave(handoff.repoPath) : undefined; await registerRepo(handoff.repoPath, metadata, { name: options.registryName, allowDuplicateName: options.allowDuplicateName, @@ -1604,7 +1605,6 @@ export async function continuePreparedAnalysis( allowDuplicateName: options.allowDuplicateName, }); } else if (!target.sharedStore) { - const prior = options.noShare ? await optedInSlotToLeave(handoff.repoPath) : undefined; if (prior) await leaveSharedStore(handoff.repoPath, prior, target.storagePath, log); else await registerLeftStore(handoff.repoPath, target.storagePath); } @@ -1736,6 +1736,7 @@ export async function runFullAnalysis( callbacks: AnalyzeCallbacks, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { + callbacks.signal?.throwIfAborted(); if (options.graphPublication && !options.resources) throw new Error('Graph publication requires admitted resources'); if (options.graphPublication === 'after-exit' && !options.skipNativeCloseOnExit) @@ -1828,6 +1829,7 @@ export async function runFullAnalysis( if (options.graphPublication === 'auto') { const prior = await loadMeta(writeTarget.metaDir); const separate = Boolean( + (await retainsPreparedGraph(writeTarget.metaDir)) || prior?.graphGeneration || prior?.embeddingJob || (!options.dropEmbeddings && prior?.embeddingCheckpoint) || @@ -1846,12 +1848,13 @@ export async function runFullAnalysis( // can only belong to an interrupted predecessor; sibling stages live // under their own meta directories and must remain untouched. Reclaim // before freshness/FTS-only early returns as well as parsing runs. + const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined; if (options.graphPublication) { const recovered = await recoverPreparedGraph(writeTarget, repoPath, options, log); if (recovered?.graphPath) writeTarget = { ...writeTarget, - lbugPath: path.join(writeTarget.metaDir, recovered.graphPath), + lbugPath: resolveGraphWritePath(writeTarget.metaDir), }; } await removeColdParseRebuildDir(getColdParseRebuildDir(writeTarget.metaDir), false); @@ -1874,7 +1877,6 @@ export async function runFullAnalysis( }); } sweepStagingArtifacts(writeTarget.metaDir, log); - const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined; const result = await runFullAnalysisInner( repoPath, options, @@ -1912,16 +1914,21 @@ export async function runFullAnalysis( options.graphPublication === 'inline' || (options.graphPublication === 'after-exit' && result.alreadyUpToDate) ) { - const published = await loadMeta(writeTarget.metaDir); + let published = await loadMeta(writeTarget.metaDir); const generate = _deriveEmbeddingMode( options, published?.stats?.embeddings ?? 0, ).shouldGenerateEmbeddings; if (published && (generate || published.embeddingJob || published.embeddingCheckpoint)) { + if (flatShared && !published.graphGeneration) { + if (!(await ensurePrivateSharedGraph(writeTarget.metaDir, log))) + throw new Error('The shared graph is unavailable; re-run analyze to rebuild it.'); + published = (await loadMeta(writeTarget.metaDir)) ?? published; + } const { runAnalysisEmbeddingPhase } = await import('./analysis-embedding-phase.js'); const metadata = await runAnalysisEmbeddingPhase({ metadataDirectory: writeTarget.metaDir, - graphPath: path.resolve(writeTarget.metaDir, published.graphPath ?? 'lbug'), + graphPath: resolveGraphPath(writeTarget.metaDir, published), resources: options.resources!, callbacks, nodeLimit: options.embeddingsNodeLimit, @@ -1995,7 +2002,7 @@ async function matchesIndexedDirtyWorktree(repoPath: string, meta: RepoMeta): Pr !Array.isArray(indexedDirtyPaths) || indexedDirtyPaths.length === 0 || indexedDirtyPaths.some((rel) => typeof rel !== 'string') || - getIndexIncompleteReasons(meta).length > 0 || + getGraphIncompleteReasons(meta).length > 0 || indexCoverage?.maxFileSizeBytes !== getMaxFileSizeBytes() ) { return false; @@ -3642,6 +3649,7 @@ async function runFullAnalysisInner( inputs.map((file) => file.path), ); } + callbacks.signal?.throwIfAborted(); pipelineResult = await runPipelineFromRepo( repoPath, (p) => { @@ -3653,6 +3661,7 @@ async function runFullAnalysisInner( progress(p.phase, scaled, message); }, { + signal: callbacks.signal, parseCache, parsedFileStorePath: coldParseRebuildDir, parseWorkerUrl: parseIdentityResolution.ok ? parseIdentityResolution.workerUrl : undefined, @@ -3723,6 +3732,7 @@ async function runFullAnalysisInner( } // ── Phase 2: LadybugDB (60–85%) ────────────────────────────────── + callbacks.signal?.throwIfAborted(); progress('lbug', 60, 'Loading into LadybugDB...'); // Parsing and graph construction always see the original source. Apply the @@ -4964,6 +4974,7 @@ async function runFullAnalysisInner( } // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── + callbacks.signal?.throwIfAborted(); // The analyze (write) path owns building the search indexes, so it uses // the `auto` install policy (LOAD-first, then one bounded INSTALL) — // symmetric with the VECTOR/embeddings path below and consistent with the @@ -5700,6 +5711,7 @@ async function runFullAnalysisInner( } // ── Phase 5: Finalize (98–100%) ─────────────────────────────────── + callbacks.signal?.throwIfAborted(); progress('done', 98, 'Saving metadata...'); // Count embeddings in the index (cached + newly generated). Tri-state, and diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 8168a1557..25d5565f4 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -61,6 +61,7 @@ import { import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { wipeLbugDbFiles } from './lbug/lbug-adapter.js'; import { inspectLbugSidecars } from './lbug/sidecar-recovery.js'; +import { getExactScanLimit } from './platform/capabilities.js'; type Log = (msg: string) => void; @@ -177,7 +178,7 @@ async function retainSharedMetadata( } exported.stats = { ...meta.stats, embeddings: completed ? vector.chunks : 0 }; if (meta.capabilities) { - const exactScanLimit = meta.capabilities.vectorSearch.exactScanLimit; + const exactScanLimit = meta.capabilities.vectorSearch?.exactScanLimit ?? getExactScanLimit(); const readable = completed && vector.chunks > 0 && diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 0c1423217..c55869f83 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -4607,6 +4607,9 @@ export class LocalBackend { getExactScanLimit(), ); if (selection.kind === 'unavailable') { + if (degraded && selection.reason === 'embedding-pending') + degraded.reason = + 'Semantic search is unavailable while embeddings are pending. Run gitnexus analyze to resume; graph and keyword search remain available.'; this.lastQueryEmbeddingDims.delete(repo.lbugPath); return []; } diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index df1f1ebee..b82727509 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -90,7 +90,7 @@ const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const JOB_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes /** How long to retain event-loop references after requesting safe cancellation. */ -const CANCEL_GRACE_MS = 15_000; +const CANCEL_GRACE_MS = 60_000; export class JobManager { private jobs = new Map(); @@ -334,7 +334,7 @@ export class JobManager { } /** - * Request cancellation, then abandon event-loop references after grace. + * Request cancellation, then stop a wedged standalone process after grace. * Ownership and the job slot remain held until the actual process exit. */ private requestChildShutdown(jobId: string, child: ChildProcess): void { @@ -351,8 +351,9 @@ export class JobManager { const grace = setTimeout(() => { this.cancelGraceTimers.delete(jobId); if (child.exitCode === null && child.signalCode === null) { - child.channel?.unref(); - child.unref?.(); + // Process termination preserves crash-recovery receipts. Never use + // this policy for worker_threads running native code in this process. + child.kill('SIGKILL'); } }, CANCEL_GRACE_MS); grace.unref?.(); @@ -360,14 +361,10 @@ export class JobManager { } dispose() { - // Never turn server shutdown into a native-active worker kill. + // Cooperative cancellation precedes bounded standalone-process shutdown. for (const [jobId, child] of this.children) { this.requestChildShutdown(jobId, child); - child.channel?.unref(); - child.unref?.(); } - for (const timer of this.cancelGraceTimers.values()) clearTimeout(timer); - this.cancelGraceTimers.clear(); for (const controller of this.abortControllers.values()) controller.abort(); this.abortControllers.clear(); @@ -381,6 +378,19 @@ export class JobManager { this.emitter.removeAllListeners(); } + /** Keep shutdown alive until standalone writers have physically exited. */ + async waitForChildren(): Promise { + await Promise.all( + [...this.children.values()].map( + (child) => + new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) resolve(); + else child.once('exit', () => resolve()); + }), + ), + ); + } + private isTerminal(status: AnalyzeJob['status']): boolean { return isTerminalJobStatus(status); } @@ -392,7 +402,12 @@ export class JobManager { private cleanup() { const now = Date.now(); for (const [id, job] of this.jobs) { - if (this.isTerminal(job.status) && job.completedAt && now - job.completedAt > JOB_TTL_MS) { + if ( + !this.children.has(id) && + this.isTerminal(job.status) && + job.completedAt && + now - job.completedAt > JOB_TTL_MS + ) { this.jobs.delete(id); this.abortControllers.delete(id); } diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 8891369b2..bd2639880 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -47,6 +47,7 @@ import { executePrepared, streamQuery, closeLbug, + drainAnalysisNativeResources, withLbugDb, isReadOnlyDbError, } from '../core/lbug/lbug-adapter.js'; @@ -2212,12 +2213,22 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => signal: embedController.signal, callbacks: { onGraphPublished: async () => { - await closeLbug(); + await drainAnalysisNativeResources(); await backend.init(); }, onProgress: (phase, percent, message, details) => { embedJobManager.updateJob(job.id, { - progress: { ...projectAnalysisPhaseProgress(details), phase, percent, message }, + progress: { + ...projectAnalysisPhaseProgress(details), + phase: + phase === 'embeddings' + ? details?.embeddingProgress?.phase === 'indexing' + ? 'indexing' + : 'embedding' + : phase, + percent, + message, + }, }); }, onLog: (message) => logger.info({ repo: entry.name }, message), @@ -2399,6 +2410,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // Abort requests are cooperative. Keep the ordinary writer locks and // native handles alive until each in-process embedding run has settled. await Promise.allSettled(activeEmbeddingRuns); + await jobManager.waitForChildren(); await cleanupMcp(); await closeLbug(); await backend.disconnect(); diff --git a/gitnexus/src/storage/analysis-artifact-cleanup.ts b/gitnexus/src/storage/analysis-artifact-cleanup.ts new file mode 100644 index 000000000..4742eae7b --- /dev/null +++ b/gitnexus/src/storage/analysis-artifact-cleanup.ts @@ -0,0 +1,120 @@ +/** Reclaim only unreferenced owned artifacts under the ordinary slot lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { RepoMeta } from './repo-meta.js'; +import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js'; +import { readEmbeddingArtifactOwners } from './embedding-artifact.js'; +import { readLegacyEmbeddingSource } from './legacy-embedding-source.js'; +import { logger } from '../core/logger.js'; + +const GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/; +const ID = /^[a-f0-9]{64}$/; + +/** Keep the published and previous graph, all model spaces, pending jobs, and + * their paid reuse sources. Windows busy files stay in place for the next run. */ +export async function reclaimAnalysisArtifacts( + directory: string, + metadata: RepoMeta, + previousGraph?: string, + log?: (message: string) => void, +): Promise { + const warn = log ?? ((message: string) => logger.warn(message)); + try { + const root = await embeddingJobDirectory(directory); + const entries = await fs.readdir(root, { withFileTypes: true }); + if (entries.some((entry) => entry.name === 'analysis-publication.json')) return; + const graphs = new Set(); + const jobs = new Set(); + const legacy = new Set(); + const keepGraph = (file: string | undefined) => { + if (file && GRAPH.test(file)) graphs.add(file); + }; + const keepJob = (id: string | undefined) => { + if (!id) return; + if (!ID.test(id)) throw new Error('Invalid referenced embedding identity'); + jobs.add(id); + }; + keepGraph(metadata.graphPath); + keepGraph(previousGraph); + if (metadata.legacyVectorSource) legacy.add(metadata.legacyVectorSource); + for (const reference of [metadata.embeddingJob, metadata.vectorGeneration]) { + if (!reference) continue; + keepJob(reference.identity); + keepJob(reference.artifactIdentity); + } + for (const owner of await readEmbeddingArtifactOwners(root)) { + keepJob(owner.artifactIdentity); + keepJob(owner.jobIdentity); + keepJob(owner.fallbackArtifact); + keepGraph(`lbug.generation.${owner.graphGeneration}`); + if (owner.fallbackLegacy) legacy.add(owner.fallbackLegacy); + } + // Validate every reference before the first deletion, including retired + // legacy graphs that still contain the only copy of paid vectors. + for (const identity of legacy) { + const source = await readLegacyEmbeddingSource(root, identity); + keepGraph(source.sourceFile); + } + const embeddingRoot = path.join(root, 'embeddings'); + let jobEntries: string[] = []; + try { + await embeddingJobDirectory(embeddingRoot); + jobEntries = await fs.readdir(embeddingRoot); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const orphanJobs: string[] = []; + for (const name of jobEntries) { + if (typeof name !== 'string' || !ID.test(name) || jobs.has(name)) continue; + const job = await embeddingJobDirectory(path.join(embeddingRoot, name)); + try { + const receipt = (await readEmbeddingJobRecord(job, 'job.json')) as { + version?: number; + identity?: string; + }; + if (receipt.version !== 1 || receipt.identity !== name) + throw new Error('Unreferenced embedding job has unknown ownership'); + orphanJobs.push(job); + } catch (error) { + // An interrupted bootstrap has no paid work, but without a receipt we + // leave it to explicit cleanup instead of guessing ownership. + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + const remove = async (file: string, recursive = false): Promise => { + try { + await fs.rm(file, { force: true, recursive }); + return true; + } catch (error) { + warn( + `Retained busy analysis artifact ${path.basename(file)}; cleanup will retry (${(error as Error).message}).`, + ); + return false; + } + }; + for (const entry of entries) { + if (!entry.isFile() || !GRAPH.test(entry.name) || graphs.has(entry.name)) continue; + if (await remove(path.join(root, entry.name))) + for (const suffix of ['.wal', '.shadow']) + await remove(path.join(root, entry.name + suffix)); + } + for (const job of orphanJobs) { + // Keep the ownership receipt until every payload is gone. Recursive + // removal of the whole directory can delete job.json before a busy + // native file fails on Windows, making the next cleanup unverifiable. + let removed = true; + for (const name of await fs.readdir(job)) { + if (name === 'job.json') continue; + if (!(await remove(path.join(job, name), true))) { + removed = false; + break; + } + } + if (removed && (await remove(path.join(job, 'job.json')))) await fs.rmdir(job); + } + } catch (error) { + warn( + `Retained analysis artifacts because ownership could not be verified (${(error as Error).message}).`, + ); + } +} diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts index e997fd02b..3e4dea315 100644 --- a/gitnexus/src/storage/embedding-artifact.ts +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -39,6 +39,47 @@ export interface EmbeddingArtifactRecord { owner: EmbeddingArtifactOwner; } +/** Read every model space before reclamation. An interrupted ownership change + * retains both sides; malformed/unreadable receipts abort the entire sweep. */ +export async function readEmbeddingArtifactOwners(root: string): Promise { + const parent = path.join(root, 'embeddings', 'spaces'); + try { + await fs.lstat(parent); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + await embeddingJobDirectory(path.dirname(parent)); + await embeddingJobDirectory(parent); + const owners: EmbeddingArtifactOwner[] = []; + for (const entry of await fs.readdir(parent, { withFileTypes: true })) { + if (!DIGEST.test(entry.name)) continue; + const directory = await embeddingJobDirectory(path.join(parent, entry.name)); + const add = (value: unknown) => { + const raw = value as EmbeddingArtifactRecord; + const validated = record(raw, raw?.space); + if (validated.spaceKey !== entry.name) throw new Error('Embedding space directory mismatch'); + owners.push(validated.owner); + }; + const current = await optionalRecord(directory, 'space.json'); + if (current !== null) add(current); + const intent = (await optionalRecord(directory, 'rebase.json')) as { + version: number; + before: unknown; + after: unknown; + } | null; + if (intent !== null) { + if (intent.version !== 1 || intent.before === undefined || !intent.after) + throw new Error('Invalid embedding rebase intent; artifacts retained'); + if (intent.before !== null) add(intent.before); + add(intent.after); + } + if (current === null && intent === null) + throw new Error('Embedding space has no ownership receipt'); + } + return owners; +} + type ValidateOwner = (owner: EmbeddingArtifactOwner) => Promise; const DIGEST = /^[a-f0-9]{64}$/; diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts index 58fa75e8c..b62b980cb 100644 --- a/gitnexus/src/storage/embedding-job-journal.ts +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -18,6 +18,8 @@ export interface EmbeddingUnitIntent { next: EmbeddingManifestCursor; nodes: number; origin: 'manifest' | 'failed'; + /** Bounded by the unit's 64 nodes; three deterministic rejections exclude a node. */ + inputFailures?: Record; } export interface EmbeddingJobState { @@ -34,6 +36,7 @@ export interface EmbeddingJobState { reusedNodes: number; failedUnits: number; failedNodes: number; + excludedNodes?: number; chunks: number; active: EmbeddingUnitIntent | null; phase: 'embedding' | 'indexing' | 'ready'; @@ -84,6 +87,22 @@ function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): !['manifest', 'failed'].includes(intent.origin) ) throw new Error('Invalid embedding unit intent'); + if ( + intent.inputFailures !== undefined && + (!intent.inputFailures || + typeof intent.inputFailures !== 'object' || + Array.isArray(intent.inputFailures) || + Object.keys(intent.inputFailures).length > intent.nodes || + Object.entries(intent.inputFailures).some( + ([id, attempts]) => + !id.length || + id.length > 1024 * 1024 || + !integer(attempts) || + attempts < 1 || + attempts > 3, + )) + ) + throw new Error('Invalid embedding input rejection history'); } function validateState( @@ -115,9 +134,10 @@ function validateState( s.failedUnits, s.failedNodes, s.chunks, + s.excludedNodes ?? 0, ].every(integer) || s.completedUnits + s.reusedUnits + s.failedUnits > s.totalUnits || - s.completedNodes + s.reusedNodes + s.failedNodes > s.totalNodes || + s.completedNodes + s.reusedNodes + s.failedNodes + (s.excludedNodes ?? 0) > s.totalNodes || !['embedding', 'indexing', 'ready'].includes(s.phase) || s.active === undefined ) @@ -139,7 +159,7 @@ function validateState( (s.active !== null || s.failedUnits !== 0 || settled !== s.totalUnits || - s.completedNodes + s.reusedNodes !== s.totalNodes)) + s.completedNodes + s.reusedNodes + (s.excludedNodes ?? 0) !== s.totalNodes)) ) throw new Error('Embedding job accounting is inconsistent'); } @@ -310,6 +330,7 @@ export class EmbeddingJobJournal { throw new Error('Failed embedding unit cursor mismatch'); next.failedUnits--; next.failedNodes -= requested.nodes; + requested.inputFailures = record.intent.inputFailures; } next.active = requested; }); @@ -319,10 +340,13 @@ export class EmbeddingJobJournal { async acknowledge(result: { reusedNodes: number; chunks: number }): Promise { await this.mutate(async (next) => { const active = next.active; + const excluded = Object.values(active?.inputFailures ?? {}).filter( + (attempts) => attempts >= 3, + ).length; if ( !active || !integer(result.reusedNodes) || - result.reusedNodes > active.nodes || + result.reusedNodes > active.nodes - excluded || !integer(result.chunks) ) throw new Error('Invalid embedding unit acknowledgement'); @@ -333,7 +357,8 @@ export class EmbeddingJobJournal { if (result.reusedNodes === active.nodes) next.reusedUnits++; else next.completedUnits++; next.reusedNodes += result.reusedNodes; - next.completedNodes += active.nodes - result.reusedNodes; + next.completedNodes += active.nodes - result.reusedNodes - excluded; + if (excluded) next.excludedNodes = (next.excludedNodes ?? 0) + excluded; next.chunks += result.chunks; if (active.origin === 'manifest') next.cursor = active.next; next.active = null; @@ -363,6 +388,23 @@ export class EmbeddingJobJournal { }); } + /** Persist input-only failures before discarding rejected nodes' partial rows. */ + async recordInputRejections(ids: readonly string[]): Promise>> { + if (!ids.length) return clone(this.state.active?.inputFailures ?? {}); + await this.mutate(async (next) => { + if (!next.active || new Set(ids).size !== ids.length || ids.length > next.active.nodes) + throw new Error('Invalid embedding input rejections'); + const history = { ...next.active.inputFailures }; + for (const id of ids) { + if (!id.length || id.length > 1024 * 1024 || id === '__proto__') + throw new Error('Invalid rejected node identity'); + history[id] = Math.min(3, (history[id] ?? 0) + 1); + } + next.active.inputFailures = history; + }); + return clone(this.state.active!.inputFailures!); + } + private async readFailed(name: string): Promise { const record = (await readEmbeddingJobRecord(this.directory, name)) as FailedUnit; if ( @@ -395,10 +437,10 @@ export class EmbeddingJobJournal { const entries = await fs.opendir(this.directory); for await (const entry of entries) { if (!FAILED.test(entry.name)) continue; + const record = await this.readFailed(entry.name); // An active retry owns this unit even if its failure file was already // removed, or a crash left the previous failed record beside it. if (this.state.active && entry.name === filename(this.state.active.start.unit)) continue; - const record = await this.readFailed(entry.name); units++; nodes += record.intent.nodes; if ( diff --git a/gitnexus/src/storage/embedding-job-progress.ts b/gitnexus/src/storage/embedding-job-progress.ts index 7cb8da501..dee2e2345 100644 --- a/gitnexus/src/storage/embedding-job-progress.ts +++ b/gitnexus/src/storage/embedding-job-progress.ts @@ -16,7 +16,11 @@ export async function readEmbeddingJobProgress( ): Promise { const reference = metadata?.embeddingJob; if (!reference) { - if (!metadata?.vectorGeneration?.preservedForGraph) return undefined; + if ( + !metadata?.vectorGeneration?.preservedForGraph && + !metadata?.vectorGeneration?.excludedNodes + ) + return undefined; const selected = selectVectorGeneration( metadataDirectory, metadata, @@ -25,8 +29,14 @@ export async function readEmbeddingJobProgress( ); // These are historical chunks, not completed/reused work in a current // manifest. Unknown selection totals stay absent, never zero or complete. - return selected.kind === 'ready' && selected.coverage === 'preserved' - ? { phase: 'preserved', chunks: selected.generation.chunks } + return selected.kind === 'ready' + ? { + phase: selected.coverage === 'preserved' ? 'preserved' : 'ready', + chunks: selected.generation.chunks, + ...(selected.generation.excludedNodes + ? { excludedNodes: selected.generation.excludedNodes } + : {}), + } : undefined; } try { diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts index 8f7656145..82b310325 100644 --- a/gitnexus/src/storage/index-capabilities.ts +++ b/gitnexus/src/storage/index-capabilities.ts @@ -9,7 +9,13 @@ export async function readIndexCapabilities( directory: string, metadata: RepoMeta | null | undefined, ): Promise { - if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob) + if ( + !metadata?.graphGeneration && + !metadata?.vectorGeneration && + !metadata?.embeddingJob && + !metadata?.incrementalInProgress && + !metadata?.graphWriteCollapsed + ) return metadata?.capabilities; const exactScanLimit = getExactScanLimit(); let vectorSearch: NonNullable['vectorSearch']; @@ -31,6 +37,7 @@ export async function readIndexCapabilities( exactScanLimit, coverage: selection.coverage ?? 'current', sourceGraphGeneration: selection.generation.graphGeneration, + ...(selection.generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}), }; } else if (selection.kind === 'legacy' && metadata?.capabilities?.vectorSearch) { vectorSearch = { ...metadata.capabilities.vectorSearch, coverage: 'preserved' }; diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index 486d21d91..6a146fa4b 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -226,7 +226,7 @@ const unverifiedGuardError = (guardPath: string): Error => const HOSTNAME = os.hostname(); /** Linux: field 22 of /proc//stat (starttime). null elsewhere / on error. */ -const readProcStartTime = (pid: number): string | null => { +export const readProcStartTime = (pid: number): string | null => { if (process.platform !== 'linux') return null; try { const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); @@ -453,7 +453,7 @@ export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => vo } // Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.`, `lbug.staging..wal`, … // Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`. - const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/; + const stagingRe = /^lbug\.(?:generation\.[a-f0-9]{32}\.)?(staging\..+|new(\..+)?)$/; const retained = new Set(readEmbeddingRecovery(lockDir)?.familyFiles ?? []); let removed = 0; let entries: string[]; diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 5f4a3ef84..8c095aff4 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -154,6 +154,9 @@ export interface RepoMeta { dimensions: number; nodes: number; chunks: number; + /** Deterministic input failures exhausted three attempts in this job. */ + excludedNodes?: number; + exclusionReason?: 'input-rejected'; indexKind: 'hnsw' | 'exact'; }; lastCommit: string; diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index 0036611fb..a671afd52 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -102,6 +102,10 @@ export function selectVectorGeneration( !nonnegative(vector.dimensions) || !vector.dimensions || !nonnegative(vector.nodes) || + (vector.excludedNodes !== undefined && + (!nonnegative(vector.excludedNodes) || + vector.excludedNodes > vector.nodes || + vector.exclusionReason !== 'input-rejected')) || !nonnegative(vector.chunks) || vector.chunks < vector.nodes || (vector.indexKind !== 'hnsw' && vector.indexKind !== 'exact') diff --git a/gitnexus/test/helpers/analysis-process-exit.ts b/gitnexus/test/helpers/analysis-process-exit.ts new file mode 100644 index 000000000..0ba660b42 --- /dev/null +++ b/gitnexus/test/helpers/analysis-process-exit.ts @@ -0,0 +1,31 @@ +import type { ChildProcess } from 'node:child_process'; + +/** Observe real child exit in publication fixtures; IPC is not release evidence. */ +export function waitForAnalysisProcessExit(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + child.removeListener('exit', onExit); + child.removeListener('error', onError); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + if (code === 0 && signal === null) resolve(); + else + reject( + new Error( + `Graph process exited unsuccessfully (${signal ?? code ?? 'unknown'}); prepared artifacts retained`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + if (child.exitCode !== null || child.signalCode !== null) { + onExit(child.exitCode, child.signalCode); + return; + } + child.once('exit', onExit); + child.once('error', onError); + }); +} diff --git a/gitnexus/test/helpers/prepared-analysis-runner.mjs b/gitnexus/test/helpers/prepared-analysis-runner.mjs index 7bfc12151..cedbb9fa7 100644 --- a/gitnexus/test/helpers/prepared-analysis-runner.mjs +++ b/gitnexus/test/helpers/prepared-analysis-runner.mjs @@ -15,7 +15,7 @@ const phases = []; const result = await runFullAnalysis( process.argv[2], { - graphPublication: recovering ? 'inline' : 'after-exit', + graphPublication: recovering ? 'auto' : 'after-exit', skipNativeCloseOnExit: !recovering, resources, skipFts: true, @@ -38,11 +38,11 @@ if (recovering) { process.exit(0); } if (!result.preparedGraph) throw new Error('Expected a prepared graph'); +process.once('message', (message) => { + if (message === 'exit') process.exit(0); +}); await new Promise((resolve, reject) => process.send({ handoff: result.preparedGraph, stats: result.stats }, (error) => error ? reject(error) : resolve(), ), ); -process.once('message', (message) => { - if (message === 'exit') process.exit(0); -}); diff --git a/gitnexus/test/integration/analysis-embedding-flow.test.ts b/gitnexus/test/integration/analysis-embedding-flow.test.ts index b4b2c16c7..0859e5abe 100644 --- a/gitnexus/test/integration/analysis-embedding-flow.test.ts +++ b/gitnexus/test/integration/analysis-embedding-flow.test.ts @@ -36,6 +36,8 @@ it('publishes before HTTP work and resumes without parsing or repeating durable await fs.writeFile(path.join(repo, 'a.ts'), content); git('add', 'a.ts'); git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + // Retrying pending embeddings must also reuse an unchanged dirty graph. + await fs.writeFile(path.join(repo, 'a.ts'), `${content}\n// dirty snapshot\n`); const accepted: string[] = []; let fail = true; let requests = 0; @@ -102,10 +104,19 @@ it('publishes before HTTP work and resumes without parsing or repeating durable embeddingsNodeLimit: 0, }; const phases: string[] = []; + let graphPublished!: () => void; + const published = new Promise((resolve) => { + graphPublished = resolve; + }); + let releasePublication!: () => void; + const publicationHeld = new Promise((resolve) => { + releasePublication = resolve; + }); const callbacks = { async onGraphPublished() { // A serving frontend can finish its asynchronous reader refresh first. - await new Promise((resolve) => setTimeout(resolve, 5)); + graphPublished(); + await publicationHeld; graphWasVisible = true; }, onProgress(phase: string) { @@ -113,7 +124,12 @@ it('publishes before HTTP work and resumes without parsing or repeating durable if (phase === 'graph-ready') graphWasReady = true; }, }; - await expect(runFullAnalysis(repo, options, callbacks)).rejects.toThrow(/unfinished|incomplete/); + const firstRun = runFullAnalysis(repo, options, callbacks); + void firstRun.catch(() => {}); + await Promise.race([published, firstRun]); + expect(requests).toBe(0); + releasePublication(); + await expect(firstRun).rejects.toThrow(/unfinished|incomplete/); expect(endpointErrors).toEqual([]); expect(accepted).toHaveLength(64); const interrupted = await loadMeta(slot); diff --git a/gitnexus/test/integration/analysis-publication-flow.test.ts b/gitnexus/test/integration/analysis-publication-flow.test.ts index dd0e1fe84..a6f5cc657 100644 --- a/gitnexus/test/integration/analysis-publication-flow.test.ts +++ b/gitnexus/test/integration/analysis-publication-flow.test.ts @@ -12,10 +12,8 @@ import { readPreparedAnalysis, publishPreparedAnalysis, } from '../../src/core/analysis-publication.js'; -import { - waitForAnalysisProcessExit, - type PreparedAnalysisHandoff, -} from '../../src/core/analysis-phase-supervisor.js'; +import type { PreparedAnalysisHandoff } from '../../src/core/analysis-phase-supervisor.js'; +import { waitForAnalysisProcessExit } from '../helpers/analysis-process-exit.js'; import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; import type { RepoMeta } from '../../src/storage/repo-meta.js'; import lbug from '@ladybugdb/core'; @@ -224,6 +222,22 @@ it('advances the logical generation on a tiny edit without copying its private g expect((await meta()).fileHashes?.['b.ts']).toBeDefined(); }, 120_000); +it('verifies published FTS with an empty extension home', async () => { + const home = path.join(root, 'empty-home'); + await fs.mkdir(home); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + await runFullAnalysis(repo, { ...options, skipFts: false }, callbacks); + const published = await meta(); + expect(published.capabilities?.fts.status).toBe('available'); + expect(published.graphGeneration).toBeDefined(); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect(fs.stat(path.join(home, '.lbdb', 'extension'))).rejects.toMatchObject({ + code: 'ENOENT', + }); +}, 120_000); + it('preserves completed vector coverage across a new graph generation', async () => { await runFullAnalysis(repo, options, callbacks); const first = await meta(); diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index aa98c6964..eb02c2022 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -394,6 +394,11 @@ describe('pending artifacts across graph generations', () => { expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); const pending = (await loadMeta(f.root))?.embeddingJob; expect(pending?.artifactIdentity).not.toBe(fallback.artifactIdentity); + expect( + ( + await fs.stat(path.join(f.root, 'embeddings', fallback.artifactIdentity, 'vectors')) + ).isFile(), + ).toBe(true); changed.add('Const:a.ts:n00001'); await f.nextGeneration(); f.provider.mockClear(); @@ -403,7 +408,7 @@ describe('pending artifacts across graph generations', () => { expect(final.metadata.vectorGeneration?.artifactIdentity).toBe(pending?.artifactIdentity); await expect( fs.stat(path.join(f.root, 'embeddings', fallback.artifactIdentity, 'vectors')), - ).resolves.toBeDefined(); + ).rejects.toMatchObject({ code: 'ENOENT' }); }); it('uses a sealed but unpublished artifact as a read-only fallback after a graph edit', async () => { @@ -660,11 +665,21 @@ describe('published graph embedding job controller', () => { expect(f.provider).not.toHaveBeenCalled(); expect(f.initializeProvider).not.toHaveBeenCalled(); if (!first.metadata.vectorGeneration) throw new Error('Expected published vectors'); + await expect( + fs.stat(path.join(f.root, 'embeddings', first.metadata.vectorGeneration.identity, 'vectors')), + ).rejects.toMatchObject({ code: 'ENOENT' }); expect( - await fs.stat( - path.join(f.root, 'embeddings', first.metadata.vectorGeneration.identity, 'vectors'), - ), - ).toBeDefined(); + ( + await fs.stat( + path.join( + f.root, + 'embeddings', + next.metadata.vectorGeneration!.artifactIdentity!, + 'vectors', + ), + ) + ).isFile(), + ).toBe(true); }); it('does no paid replay if the process stops after publishing the vector pointer', async () => { diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 35555b9ac..5d473c95a 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs/promises'; import lbug from '@ladybugdb/core'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { VectorStore, type VectorStoreOptions } from '../../src/core/lbug/vector-store.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; @@ -700,6 +701,37 @@ describe('bounded durable embedding runner', () => { }); }); + it('excludes a permanently rejected node after three invocations and seals its healthy siblings', async () => { + const fixture = await runnerFixture(12); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00003'))) + throw new HttpEmbeddingError('bad input', { status: 422 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + await fixture.run(); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).indexKind).toBeUndefined(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + await fixture.reopen(); + fixture.provider.mockClear(); + const final = await fixture.run(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(final.state).toMatchObject({ + failedUnits: 0, + excludedNodes: 1, + reusedNodes: 11, + completedNodes: 0, + chunks: 11, + phase: 'indexing', + }); + expect(final.indexKind).toBe('exact'); + expect(store.isSealed).toBe(true); + fixture.provider.mockClear(); + await fixture.run(); + expect(fixture.provider).not.toHaveBeenCalled(); + }); + it('bounds one oversized node by bytes and preserves all generated Unicode input', async () => { const content = '漢😀abcdef'.repeat(60000); const fixture = await runnerFixture(1, content); @@ -983,7 +1015,137 @@ describe('separate durable vector artifacts', () => { }); }); +async function loadVectorFixture(directory: string): Promise { + const probe = await openLbugConnection(lbug, path.join(directory, 'extension-probe'), { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + return await new ExtensionManager().ensure( + async (sql) => { + const result = await probe.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) { + try { + await cursor.getAll(); + } finally { + cursor.close(); + } + } + }, + 'vector', + 'vector fixture', + { policy: 'load-only' }, + ); + } finally { + await closeLbugConnection(probe); + } +} + describe('published vector readers', () => { + it('seals a 20001-row 384-dimensional artifact within the admitted writer pool', async (context) => { + const initial = await setup(); + await store.close(); + await fs.rm(path.join(initial.directory, 'vectors'), { force: true }); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + const loaded = await loadVectorFixture(initial.directory); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } + expect(loaded).toBe(true); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + store = await VectorStore.open({ + ...initial, + dimensions: 384, + batchBytes: 2 * 1024 ** 2, + // The controller reserves the other half for a compatible reuse reader. + bufferPoolBytes: Math.floor(resources.embedding.vectorPoolBytes / 2), + }); + const nodeId = 'Function:source.ts:large'; + await store.beginNode(nodeId, 'hash'); + const total = 20001; + for (let start = 0; start < total; start += 256) { + await store.append( + nodeId, + 'hash', + Array.from({ length: Math.min(256, total - start) }, (_, offset) => ({ + chunkIndex: start + offset, + startLine: start + offset + 1, + endLine: start + offset + 1, + embedding: Array.from({ length: 384 }, (_, dimension) => + dimension === 0 ? 1 : ((start + offset + dimension) % 997) / 997, + ), + })), + ); + } + await store.finishNode(nodeId, 'hash', total); + const sealed = await store.seal(1, total); + expect(sealed).toBe('hnsw'); + expect(store.isSealed).toBe(true); + const options = store.options; + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.seal(1, total)).toBe('hnsw'); + }, 120_000); + + it('resumes writes after a crash between HNSW creation and publication', async (context) => { + const options = await setup(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + const nodeId = 'Function:source.ts:alpha'; + await store.beginNode(nodeId, 'old'); + await store.append(nodeId, 'old', [chunk(0)]); + await store.finishNode(nodeId, 'old', 1); + await store.checkpointUnit( + [...(await store.completedNodes([nodeId]))].map(([, receipt]) => receipt), + ); + await store.close(); + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) { + try { + await cursor.getAll(); + } finally { + cursor.close(); + } + } + }; + const loaded = await new ExtensionManager().ensure( + query, + 'vector', + 'interrupted vector fixture', + { policy: 'load-only' }, + ); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } + expect(loaded).toBe(true); + await query( + "CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle, { + strict: true, + databasePath: path.join(options.directory, 'vectors'), + }); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await store.beginNode(nodeId, 'new'); + await store.append(nodeId, 'new', [chunk(0), chunk(1)]); + await store.finishNode(nodeId, 'new', 2); + expect(await store.seal(1, 2)).toBe('hnsw'); + }); + it('shares one native database across independently pinned vector files', async () => { const options = await setup(); const id = 'Function:source.ts:alpha'; @@ -1054,6 +1216,7 @@ describe('published vector readers', () => { policy: 'load-only', }); if (!ready) { + if (process.env.GITNEXUS_REQUIRE_FTS === '1') throw new Error('Native FTS is required'); context.skip('Optional native FTS extension is unavailable'); return; } @@ -1128,22 +1291,12 @@ describe('published vector readers', () => { vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '1'); const root = temporary.dbPath; - const probe = await openLbugConnection(lbug, path.join(root, 'extension-probe'), { - bufferPoolBytes: 64 * 1024 * 1024, - threads: 1, - }); - try { - const loaded = await probe.conn.query('LOAD vector'); - for (const result of Array.isArray(loaded) ? loaded : [loaded]) { - await result.getAll(); - await result.close(); - } - } catch { + const loaded = await loadVectorFixture(root); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { context.skip('Optional native VECTOR extension is unavailable'); return; - } finally { - await closeLbugConnection(probe); } + expect(loaded).toBe(true); const generation = 'a'.repeat(32); const identity = 'b'.repeat(64); const directory = path.join(root, 'embeddings', identity); @@ -1198,6 +1351,28 @@ describe('published vector readers', () => { expect(found).toEqual([ expect.objectContaining({ nodeId: 'Function:source.ts:alpha', startLine: 1, endLine: 2 }), ]); + const execute = readerPool.executeParameterized; + vi.spyOn(readerPool, 'executeParameterized').mockImplementation((db, sql, params) => + sql.includes('QUERY_VECTOR_INDEX') ? Promise.resolve([]) : execute(db, sql, params), + ); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '10'); + expect( + await searchVectorGeneration( + { metadataDirectory: root, metadata, graphPath: path.join(root, metadata.graphPath!) }, + async () => [0.1, 0.2, 0.3, 0.4], + 5, + 1, + async () => ({ provider: 'local', model: 'fixture', dimensions: 4 }), + ), + ).toEqual([ + expect.objectContaining({ + nodeId: 'Function:source.ts:alpha', + chunkIndex: 0, + startLine: 1, + endLine: 2, + distance: expect.any(Number), + }), + ]); } finally { await readerPool.closeLbug(); } diff --git a/gitnexus/test/integration/legacy-embedding-controller.test.ts b/gitnexus/test/integration/legacy-embedding-controller.test.ts index cb83b87fb..88559ac55 100644 --- a/gitnexus/test/integration/legacy-embedding-controller.test.ts +++ b/gitnexus/test/integration/legacy-embedding-controller.test.ts @@ -240,6 +240,8 @@ describe('legacy migration through the published job controller', () => { ).rejects.toThrow('first stop'); const first = (await loadMeta(f.root))?.embeddingJob; expect(first?.artifactIdentity).toBeDefined(); + expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); + expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); await f.nextGraph(); await expect( f.run({ @@ -249,6 +251,8 @@ describe('legacy migration through the published job controller', () => { }), ).rejects.toThrow('second stop'); expect((await loadMeta(f.root))?.embeddingJob?.artifactIdentity).toBe(first?.artifactIdentity); + expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); + expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); await f.nextGraph(); const result = await f.run(); expect(result.phase).toBe('ready'); @@ -257,8 +261,7 @@ describe('legacy migration through the published job controller', () => { expect(scans).toBe(1); expect(f.provider).not.toHaveBeenCalled(); expect(f.initializeProvider).not.toHaveBeenCalled(); - expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); - expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); + await expect(fs.stat(f.sourcePath)).rejects.toMatchObject({ code: 'ENOENT' }); const owner = (await readEmbeddingJobRecord( path.join(f.root, 'embeddings', 'spaces', first?.spaceKey ?? ''), 'space.json', diff --git a/gitnexus/test/integration/legacy-vector-source.test.ts b/gitnexus/test/integration/legacy-vector-source.test.ts index 6ecfdf071..aae894a31 100644 --- a/gitnexus/test/integration/legacy-vector-source.test.ts +++ b/gitnexus/test/integration/legacy-vector-source.test.ts @@ -20,6 +20,7 @@ import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; import { isShortLabel, type EmbeddableNode } from '../../src/core/embeddings/types.js'; import { createEmbeddingWorkManifest } from '../../src/core/embeddings/embedding-work-manifest.js'; import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; import { runEmbeddingJob } from '../../src/core/embeddings/embedding-job-runner.js'; const roots: string[] = []; @@ -151,6 +152,54 @@ afterEach(async () => { }); describe('verified legacy vector reuse', () => { + it('builds a legacy reuse catalog above 50000 rows with admitted reader and writer pools', async () => { + const f = await fixture(node('Const')); + const budget = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + const pool = Math.floor(budget.embedding.vectorPoolBytes / 2); + const db = await openLbugConnection(lbug, f.options.path, { + bufferPoolBytes: pool, + threads: 1, + }); + try { + const result = await db.conn.query(`UNWIND range(1, 50000) AS i CREATE (:CodeEmbedding { + id:'Const:large.ts:n' + CAST(i AS STRING) + ':0', nodeId:'Const:large.ts:n' + CAST(i AS STRING), + chunkIndex:0,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'hash'})`); + for (const cursor of Array.isArray(result) ? result : [result]) cursor.close(); + } finally { + await closeLbugConnection(db, { strict: true, databasePath: f.options.path }); + } + await f.store.close(); + const index = await VectorStore.open({ + ...f.store.options, + bufferPoolBytes: pool, + mode: 'resume', + }); + handles.push(index); + const source = await LegacyVectorSource.open( + { ...f.options, bufferPoolBytes: pool }, + index, + () => [], + ); + handles.push(source); + for (const [nodeId, contentHash] of [ + [node('Const').id, f.hash], + ['Const:large.ts:n1', 'hash'], + ['Const:large.ts:n50000', 'hash'], + ]) + expect(await index.legacyReuseGroup(f.options.sourceIdentity, nodeId)).toMatchObject({ + nodeId, + contentHash, + chunks: 1, + }); + expect( + await index.legacyReuseGroup(f.options.sourceIdentity, 'Const:large.ts:n50001'), + ).toBeNull(); + }, 120_000); + it('relocates chunk spans without treating the last chunk as the whole node range', async () => { const original = { ...node('File', Array.from({ length: 80 }, (_, i) => `const v${i} = ${i};`).join('\n')), diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts index acca9b25d..5af29748b 100644 --- a/gitnexus/test/unit/analysis-host-resources.test.ts +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -5,7 +5,8 @@ import { } from '../../src/core/analysis-host-resources.js'; const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), placement: vi.fn() })); -vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (original) => ({ + ...(await original()), effectiveRamBytes: () => 32768 * 1024 ** 2, })); vi.mock('node:os', () => ({ default: { availableParallelism: () => 16 } })); @@ -46,7 +47,8 @@ it('reserves local model capacity only for invocations that may embed', () => { it('divides the host allowance across concurrent analyses', () => { const one = resolveHostAnalysisResources({ includeEmbeddings: false }); const four = resolveHostAnalysisResources({ includeEmbeddings: false, concurrency: 4 }); - expect(four.totalBytes * 4).toBe(one.totalBytes); + expect(four.totalBytes * 4).toBe(24576 * 1024 ** 2); + expect(four.totalBytes * 4).toBeLessThanOrEqual(one.totalBytes); expect(four.parse.nativeThreads * 4).toBe(one.parse.nativeThreads); expect(four.parse.workers).toBeLessThanOrEqual(3); }); @@ -63,6 +65,14 @@ it('honors bounded worker, pool, and embedding-thread overrides', () => { expect(budget.embedding.nativeThreads).toBe(8); }); +it('ignores embedding-only thread configuration for a graph-only invocation', () => { + vi.stubEnv('GITNEXUS_EMBEDDING_THREADS', 'not-a-number'); + expect(resolveHostAnalysisResources({ includeEmbeddings: false }).parse.workers).toBe(15); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: true })).toThrow( + 'GITNEXUS_EMBEDDING_THREADS', + ); +}); + it.each(['0', ' 0 ', '0.0'])( 'explains why the native default pool %s cannot be reserved', (value) => { diff --git a/gitnexus/test/unit/analysis-phase-supervisor.test.ts b/gitnexus/test/unit/analysis-phase-supervisor.test.ts index 287661eed..600410468 100644 --- a/gitnexus/test/unit/analysis-phase-supervisor.test.ts +++ b/gitnexus/test/unit/analysis-phase-supervisor.test.ts @@ -7,10 +7,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { projectPreparedAnalysisHandoff, publishAnalysisAfterProcessExit, - waitForAnalysisProcessExit, type AnalysisPhasePublicationActions, type PreparedAnalysisHandoff, } from '../../src/core/analysis-phase-supervisor.js'; +import { waitForAnalysisProcessExit } from '../helpers/analysis-process-exit.js'; import { readPreparedAnalysis } from '../../src/core/analysis-publication.js'; import { acquireIndexLock } from '../../src/storage/index-lock.js'; import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts index e2e94840e..b24b83340 100644 --- a/gitnexus/test/unit/analysis-publication.test.ts +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -1,7 +1,12 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; +import * as indexLock from '../../src/storage/index-lock.js'; +import { recordLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { reclaimAnalysisArtifacts } from '../../src/storage/analysis-artifact-cleanup.js'; +import { adoptEmbeddingArtifact } from '../../src/storage/embedding-artifact.js'; +import { writeEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; import { spawn } from 'node:child_process'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { @@ -43,6 +48,232 @@ afterEach(async () => { }); describe('durable graph publication', () => { + it('makes metadata durable before publishing the recovery receipt', async () => { + const { root, stage } = await fixture(); + const events: string[] = []; + const open = fs.open.bind(fs); + const rename = fs.rename.bind(fs); + const label = (file: string) => + file === root + ? 'directory' + : path.basename(file).includes('analysis-meta-') + ? 'metadata' + : 'receipt'; + vi.spyOn(fs, 'open').mockImplementation(async (...args) => { + const handle = await open(...args); + const sync = handle.sync.bind(handle); + vi.spyOn(handle, 'sync').mockImplementation(async () => { + await sync(); + events.push(`sync:${label(String(args[0]))}`); + }); + return handle; + }); + vi.spyOn(fs, 'rename').mockImplementation(async (...args) => { + await rename(...args); + events.push(`rename:${label(String(args[1]))}`); + }); + await prepareAnalysisPublication(root, stage, metadata, content); + const directorySync = process.platform === 'win32' ? [] : ['sync:directory']; + expect(events).toEqual([ + 'sync:metadata', + 'rename:metadata', + ...directorySync, + 'sync:receipt', + 'rename:receipt', + ...directorySync, + ]); + }); + + it('bounds superseded graph generations across repeated staged publications', async () => { + const { root, actions } = await fixture(); + for (let i = 0; i < 5; i++) { + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + await prepareAnalysisPublication(root, stage, (await loadMeta(root))!, content); + await publishPreparedAnalysis(root, actions); + } + const graphs = (await fs.readdir(root)).filter((name) => + /^lbug\.generation\.[a-f0-9]{32}$/.test(name), + ); + expect(graphs).toHaveLength(2); + expect(graphs).toContain((await loadMeta(root))?.graphPath); + }); + + it('reclaims unreferenced jobs while retaining all model owners and their fallback vectors', async () => { + const { root } = await fixture(); + const current = 'a'.repeat(64), + fallback = 'b'.repeat(64), + orphan = 'c'.repeat(64); + for (const id of [current, fallback, orphan]) { + const directory = path.join(root, 'embeddings', id); + await fs.mkdir(directory, { recursive: true }); + await writeEmbeddingJobRecord(directory, 'job.json', { version: 1, identity: id }); + await fs.writeFile(path.join(directory, 'vectors'), 'paid vectors'); + } + await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: current, + jobIdentity: current, + fallbackArtifact: fallback, + graphGeneration: 'f'.repeat(32), + sourceIdentity: '1'.repeat(64), + manifestDigest: '2'.repeat(64), + phase: 'pending', + revalidateInputs: false, + }, + async () => {}, + ); + await reclaimAnalysisArtifacts(root, metadata); + expect(await fs.readFile(path.join(root, 'embeddings', current, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + expect(await fs.readFile(path.join(root, 'embeddings', fallback, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + await expect(fs.stat(path.join(root, 'embeddings', orphan))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('retains an obsolete job receipt until its busy vector file can be removed', async () => { + const { root } = await fixture(); + const identity = 'e'.repeat(64); + const job = path.join(root, 'embeddings', identity); + await fs.mkdir(job, { recursive: true }); + await writeEmbeddingJobRecord(job, 'job.json', { version: 1, identity }); + const vectors = path.join(job, 'vectors'); + await fs.writeFile(vectors, 'obsolete vectors'); + const remove = fs.rm.bind(fs); + const busy = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === vectors) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + return remove(file, options); + }); + const warn = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, warn); + expect(await fs.readFile(path.join(job, 'job.json'), 'utf8')).toContain(identity); + expect(await fs.readFile(vectors, 'utf8')).toBe('obsolete vectors'); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('cleanup will retry')); + busy.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata); + await expect(fs.stat(job)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('keeps the only legacy vector source while a model-space job references it', async () => { + const { root } = await fixture(); + const sourceFile = `lbug.generation.${'c'.repeat(32)}`; + await fs.writeFile(path.join(root, sourceFile), 'paid legacy vectors'); + const legacy = await recordLegacyEmbeddingSource( + root, + { sourceFile, schemaFingerprint: 'fixture' }, + [], + ); + await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: 'a'.repeat(64), + jobIdentity: 'a'.repeat(64), + fallbackLegacy: legacy, + graphGeneration: 'f'.repeat(32), + sourceIdentity: '1'.repeat(64), + manifestDigest: '2'.repeat(64), + phase: 'pending', + revalidateInputs: false, + }, + async () => {}, + ); + await reclaimAnalysisArtifacts(root, metadata); + expect(await fs.readFile(path.join(root, sourceFile), 'utf8')).toBe('paid legacy vectors'); + }); + + it('retains everything when an ownership receipt is corrupt and retries busy files later', async () => { + const { root } = await fixture(); + const graph = path.join(root, `lbug.generation.${'b'.repeat(32)}`); + await fs.writeFile(graph, 'previous graph'); + const space = path.join(root, 'embeddings', 'spaces', 'a'.repeat(64)); + await fs.mkdir(space, { recursive: true }); + await fs.writeFile(path.join(space, 'space.json'), '{}'); + const log = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + expect(await fs.readFile(graph, 'utf8')).toBe('previous graph'); + expect(log).toHaveBeenCalledWith(expect.stringContaining('ownership could not be verified')); + await fs.rm(space, { recursive: true }); + const remove = fs.rm.bind(fs); + const blocked = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === graph) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + await remove(file, options); + }); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + expect(await fs.readFile(graph, 'utf8')).toBe('previous graph'); + blocked.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + await expect(fs.stat(graph)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('recovers a prior process incarnation even when its PID is now this process', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + const payload = { ...receipt, writerInstance: 'f'.repeat(32) }; + await fs.writeFile( + path.join(root, 'analysis-publication.json'), + JSON.stringify({ + payload, + checksum: createHash('sha256').update(JSON.stringify(payload)).digest('hex'), + }), + ); + expect((await publishPreparedAnalysis(root, actions))?.graphGeneration?.id).toBe(receipt.id); + }); + + it('does not wait for an unrelated process that reused the writer PID', async () => { + const { root, stage, actions } = await fixture(); + const start = vi.spyOn(indexLock, 'readProcStartTime').mockReturnValue('123'); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + start.mockReturnValue('456'); + expect((await publishPreparedAnalysis(root, actions))?.graphGeneration?.id).toBe(receipt.id); + }); + + it('publishes through a symlinked parent while validating the real slot', async () => { + const { root, stage } = await fixture(); + const slot = path.join(root, 'slot'); + await fs.mkdir(slot); + await fs.rename(stage, path.join(slot, path.basename(stage))); + const alias = path.join(root, 'parent-alias'); + await fs.symlink(root, alias, process.platform === 'win32' ? 'junction' : 'dir'); + const aliasedSlot = path.join(alias, 'slot'); + await saveMeta(aliasedSlot, metadata); + const receipt = await prepareAnalysisPublication( + aliasedSlot, + path.join(aliasedSlot, path.basename(stage)), + metadata, + content, + ); + const published = await publishPreparedAnalysis(aliasedSlot, { + verify: async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }, + saveMetadata: (meta) => saveMeta(aliasedSlot, meta), + }); + expect(published?.graphGeneration?.id).toBe(receipt.id); + }); + it('publishes only after the real preparing process exits', async () => { const { root, stage, actions } = await fixture(); const moduleUrl = new URL('../../dist/core/analysis-publication.js', import.meta.url).href; diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 466a9fa28..524141cc2 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -1,3 +1,4 @@ +import { heapCapMbFor } from '../../src/core/ingestion/utils/effective-ram.js'; import { describe, expect, it } from 'vitest'; import { EmbeddingByteBudget, @@ -8,6 +9,34 @@ const MIB = 1024 * 1024; const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; describe('analysis resource admission', () => { + it.each([4096, 8192, 32768])( + 'preserves automatic main-heap and worker capacity on a %s MiB host', + (memoryMb) => { + const input = { + effectiveMemoryBytes: memoryMb * MIB, + availableCpus: 8, + preferredWorkers: 16, + modelMemoryMb: 0, + }; + const graph = resolveAnalysisResourceBudget(input); + const local = resolveAnalysisResourceBudget({ ...input, modelMemoryMb: 2048 }); + const pinned = resolveAnalysisResourceBudget({ + ...input, + mainHeapMb: 12000, + allowHeapOvercommit: true, + }); + expect(graph.mainHeapMb).toBe(heapCapMbFor(memoryMb * 1024 ** 2)); + expect(graph.parse.workers).toBe(7); + expect(graph.parse.workerHeapMb).toBe( + Math.min(4096, Math.max(512, Math.floor(memoryMb / 2 / 7))), + ); + expect(local.parse).toEqual(graph.parse); + expect(pinned.parse.workers).toBe(graph.parse.workers); + expect(pinned.parse.workerHeapMb).toBe(graph.parse.workerHeapMb); + expect(local.embedding.reservedBytes).toBeLessThanOrEqual(local.totalBytes); + expect(graph.embedding.vectorPoolBytes).toBeGreaterThan(64 * MIB); + }, + ); it('preserves an explicit heap override without inflating automatic native allocations', () => { const host = { ...request, effectiveMemoryBytes: 7500 * MIB, modelMemoryMb: 0 }; const automatic = resolveAnalysisResourceBudget(host); @@ -65,10 +94,10 @@ describe('analysis resource admission', () => { includeParsing: false, }; expect( - resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 160 * MIB }).embedding + resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 96 * MIB }).embedding .reservedBytes, ).toBe(1024 * MIB); - expect(() => resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 161 * MIB })).toThrow( + expect(() => resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 97 * MIB })).toThrow( 'cannot admit', ); }); @@ -88,6 +117,7 @@ describe('analysis resource admission', () => { const budget = resolveAnalysisResourceBudget({ ...request, effectiveMemoryBytes: memoryMb * MIB, + totalMb: Math.floor(memoryMb * 0.75), availableCpus, modelMemoryMb: 0, }); @@ -124,9 +154,9 @@ describe('analysis resource admission', () => { expect(budget.parse.workerHeapMb).toBeGreaterThanOrEqual(512); expect(budget.parse.nativeThreads).toBe(availableCpus); expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); - expect(() => resolveAnalysisResourceBudget({ ...input, workerHeapMb: 4096 })).toThrow( - 'do not fit', - ); + expect(() => + resolveAnalysisResourceBudget({ ...input, totalMb: 24576, workerHeapMb: 4096 }), + ).toThrow('do not fit'); }, ); @@ -182,7 +212,9 @@ describe('analysis resource admission', () => { expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 4, workerHeapMb: 4096 }), ).toThrow('do not fit'); - expect(() => resolveAnalysisResourceBudget({ ...request, workers: 24 })).toThrow('do not fit'); + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 24 })).toThrow( + 'do not fit', + ); expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: 0 })).toThrow( 'graph pool bytes', ); diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts index 9017e0593..c1bccbf8b 100644 --- a/gitnexus/test/unit/analyze-heap-respawn.test.ts +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -151,10 +151,10 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=3916'); + expect(args).toContain('--max-old-space-size=12723'); expect(args).toContain('--max-semi-space-size=128'); - expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=3916'); - expect(opts.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB).toBe('4300'); + expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=12723'); + expect(opts.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB).toBe('13107'); expect(opts.env.NODE_OPTIONS).toContain('--max-semi-space-size=128'); expect(opts.env.GITNEXUS_RESPAWN_PROGRESS_TTY).toBe('1'); }); @@ -197,14 +197,14 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=3916'); + expect(args).toContain('--max-old-space-size=12723'); // The auto flag is appended after the ambient value, so V8's // later-flag-wins semantics resolve to the admitted whole-heap cap. - expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=3916')).toBeGreaterThan( + expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=12723')).toBeGreaterThan( opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=4096'), ); const warn = cap.records().find((r) => r.msg.includes('pins the heap to 4096MB')); - expect(warn?.msg).toContain('Re-running analyze with the admitted 4300MB main heap'); + expect(warn?.msg).toContain('Re-running analyze with the admitted 13107MB main heap'); }); it('honors GITNEXUS_MEMORY=off: keeps the small ambient heap, silently (#2649)', async () => { @@ -271,6 +271,9 @@ describe('analyzeCommand heap respawn', () => { delete process.env.NODE_OPTIONS; const child = Object.assign(new EventEmitter(), { connected: true, + exitCode: null, + signalCode: null, + kill: vi.fn(), stdout: new EventEmitter(), stderr: new EventEmitter(), send: vi.fn((_message, callback) => callback(null)), @@ -288,9 +291,12 @@ describe('analyzeCommand heap respawn', () => { { type: 'cancel' }, expect.any(Function), ); + process.emit('SIGTERM'); + expect(process.listenerCount('SIGTERM')).toBeGreaterThan(0); + expect(child.kill).not.toHaveBeenCalled(); child.emit('message', { type: ready }); await Promise.resolve(); - expect(child.send).toHaveBeenCalledTimes(2); + expect(child.send).toHaveBeenCalledTimes(3); expect(settled).toBe(false); } finally { child.emit('close', 0, null); @@ -775,7 +781,7 @@ describe('ensureHeap --memory-budget (#3137)', () => { expect({ args, source: opts.env.GITNEXUS_HEAP_LIMIT_SOURCE, records: cap.records() }).toEqual({ args: [ ...preserved, - '--max-old-space-size=3916', + '--max-old-space-size=12723', '--max-semi-space-size=128', '--stack-size=4096', ...process.argv.slice(1), diff --git a/gitnexus/test/unit/analyze-job.test.ts b/gitnexus/test/unit/analyze-job.test.ts index 77024f04f..37cba5636 100644 --- a/gitnexus/test/unit/analyze-job.test.ts +++ b/gitnexus/test/unit/analyze-job.test.ts @@ -32,6 +32,31 @@ describe('JobManager', () => { expect(retrieved).toEqual(created); }); + it('retains terminal jobs past their TTL while the child still owns the slot', () => { + vi.useFakeTimers(); + manager.dispose(); + manager = new JobManager(); + const job = manager.createJob({ repoPath: '/tmp/repo' }); + const child = Object.assign(new EventEmitter(), { + connected: true, + exitCode: null, + signalCode: null, + send: vi.fn(), + unref: vi.fn(), + }); + manager.registerChild(job.id, child as unknown as import('node:child_process').ChildProcess); + manager.updateJob(job.id, { status: 'failed', error: 'Cancelled' }); + vi.advanceTimersByTime(70 * 60 * 1000); + expect(manager.getJob(job.id)?.status).toBe('failed'); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow( + 'Analysis already in progress', + ); + child.emit('exit', 0); + vi.advanceTimersByTime(5 * 60 * 1000); + expect(manager.getJob(job.id)).toBeUndefined(); + expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); + }); + it('keeps the original deadline when a graph worker hands off to an embedding worker', () => { manager.dispose(); vi.useFakeTimers(); @@ -428,7 +453,7 @@ describe('JobManager', () => { expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('cancelJob abandons its process reference after grace, retaining ownership until exit', () => { + it('cancelJob stops a wedged process after grace, retaining ownership until exit', () => { manager.dispose(); vi.useFakeTimers(); manager = new JobManager(); @@ -455,12 +480,12 @@ describe('JobManager', () => { expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); expect(signals).toEqual([]); - expect(fakeChild.unref).toHaveBeenCalledOnce(); - expect(fakeChild.channel.unref).toHaveBeenCalledOnce(); + vi.advanceTimersByTime(45_000); + expect(signals).toEqual(['SIGKILL']); expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('dispose on Windows never force-kills a native worker', () => { + it('dispose on Windows allows cooperative shutdown before stopping a wedged process', () => { manager.dispose(); vi.useFakeTimers(); vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); @@ -518,6 +543,14 @@ describe('JobManager', () => { vi.restoreAllMocks(); }); + it('aborts in-process embedding work before shutdown waits for it', () => { + const job = manager.createJob({ repoPath: '/tmp/embed' }); + const controller = new AbortController(); + manager.registerAbortController(job.id, controller); + manager.dispose(); + expect(controller.signal.aborted).toBe(true); + }); + it('cancelJob returns false for terminal jobs', () => { const job = manager.createJob({ repoUrl: 'https://github.com/user/repo' }); manager.updateJob(job.id, { status: 'complete' }); diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 6fa9e8d77..4ce6da601 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -5,7 +5,8 @@ import os from 'node:os'; import path from 'node:path'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), effectiveRamBytes: () => 8 * 1024 ** 3, })); vi.mock('node:v8', async (importOriginal) => { @@ -113,6 +114,7 @@ vi.mock('../../src/server/analyze-job.js', () => ({ dispose() { for (const controller of this.controllers) controller.abort(); } + async waitForChildren() {} getJob() { return mocks.activeJob ?? { status: 'analyzing' }; } @@ -240,7 +242,7 @@ describe('POST /api/embed staged recovery preflight', () => { metadataDirectory: entry.storagePath, graphPath: path.join(entry.storagePath, 'lbug'), nodeLimit: 0, - resources: expect.objectContaining({ totalBytes: 6 * 1024 ** 3 }), + resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), signal: expect.any(AbortSignal), }), ); diff --git a/gitnexus/test/unit/chunker.test.ts b/gitnexus/test/unit/chunker.test.ts index af9a61c1d..aca797648 100644 --- a/gitnexus/test/unit/chunker.test.ts +++ b/gitnexus/test/unit/chunker.test.ts @@ -31,7 +31,8 @@ vi.mock('gitnexus-shared', async (importOriginal) => ({ getLanguageFromFilename, })); -import { chunkNode } from '../../src/core/embeddings/chunker.js'; +import { chunkNode, iterateNodeChunks } from '../../src/core/embeddings/chunker.js'; +import * as lineIndex from '../../src/core/embeddings/line-index.js'; type FakeNode = { type: string; @@ -250,6 +251,37 @@ describe('chunkNode', () => { expect(result[0].endLine).toBe(3); }); + it.each(['fixed', 'ast'] as const)( + 'projects %s class chunks only as the consumer requests them', + async (strategy) => { + const fields = Array.from({ length: 20 }, (_, i) => `field${i}: string;`); + const content = ['class Sample {', ...fields, '}'].join('\n'); + const tree = makeDeclarationTree('class_declaration', 'class_body', content, fields); + const expected = await chunkNode('Class', content, 'sample.ts', 10, 31, 40, 0, { + tree, + strategy, + }); + expect(expected.length).toBeGreaterThan(3); + const lines = vi.spyOn(lineIndex, 'resolveChunkLines'); + try { + const stream = iterateNodeChunks('Class', content, 'sample.ts', 10, 31, 40, 0, { + tree, + strategy, + }); + expect(lines).not.toHaveBeenCalled(); + const first = await stream.next(); + expect(first.value).toEqual(expected[0]); + // AST packing also resolves the projected line range before mapping it. + expect(lines.mock.calls.length).toBeLessThanOrEqual(2); + const actual = [first.value]; + for await (const chunk of stream) actual.push(chunk); + expect(actual).toEqual(expected); + } finally { + lines.mockRestore(); + } + }, + ); + it('preserves interface signatures via declaration-aware chunking', async () => { const content = [ 'interface Handler {', diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index 53a36d9b7..9f53ce013 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -47,6 +47,60 @@ afterEach(async () => { }); describe('durable embedding scheduling', () => { + it('retains deterministic rejection history across a crash before vector exclusion', async () => { + let journal = await job(); + const id = 'Function:a.ts:bad'; + await journal.begin(unit(0)); + for (let attempt = 1; attempt <= 3; attempt++) { + await journal.recordInputRejections([id]); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + expect(journal.snapshot().active?.inputFailures).toEqual({ [id]: attempt }); + if (attempt < 3) { + await journal.fail('provider'); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + } + } + await journal.acknowledge({ reusedNodes: 1, chunks: 1 }); + expect(journal.snapshot()).toMatchObject({ + excludedNodes: 1, + reusedNodes: 1, + completedNodes: 0, + }); + }); + + it.each([0, 4, 1.5])('rejects malformed persisted input failure count %s', async (count) => { + const journal = await job(); + await journal.begin(unit(0)); + const state = (await readEmbeddingJobRecord(journal.directory, 'job.json')) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...state, + active: { ...unit(0), inputFailures: { 'Function:a.ts:bad': count } }, + }); + await expect(EmbeddingJobJournal.open(journal.directory, identity, manifest)).rejects.toThrow(); + }); + + it('refuses a foreign failure record beside an active retry without deleting it', async () => { + const journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + await journal.begin(unit(0, 'failed')); + const name = 'failed-0000000000000000.json'; + const record = (await readEmbeddingJobRecord(journal.directory, name)) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, name, { ...record, identity: 'f'.repeat(64) }); + const before = await fs.readFile(path.join(journal.directory, name)); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toThrow('identity mismatch'); + expect(await fs.readFile(path.join(journal.directory, name))).toEqual(before); + }); + it('keeps the journal usable after validation fails before any write', async () => { const journal = await job(); await expect(journal.indexing()).rejects.toThrow('accounting'); diff --git a/gitnexus/test/unit/embedding-job-progress.test.ts b/gitnexus/test/unit/embedding-job-progress.test.ts index a3226d3a7..865593753 100644 --- a/gitnexus/test/unit/embedding-job-progress.test.ts +++ b/gitnexus/test/unit/embedding-job-progress.test.ts @@ -52,6 +52,32 @@ afterEach(async () => { }); describe('bounded durable embedding progress', () => { + it.each(['interrupted', 'collapsed'] as const)( + 'rejects a legacy vector capability after an %s graph write', + async (state) => { + const legacy: RepoMeta = { + repoPath: '/repo', + indexedAt: '', + lastCommit: '', + incrementalInProgress: + state === 'interrupted' ? { startedAt: 1, toWriteCount: 1 } : undefined, + graphWriteCollapsed: state === 'collapsed' ? { expected: 100, persisted: 1 } : undefined, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'available' }, + vectorSearch: { + provider: 'ladybugdb-vector', + status: 'vector-index', + exactScanLimit: 10000, + }, + }, + }; + expect(await readIndexCapabilities('/missing', legacy)).toMatchObject({ + vectorSearch: { status: 'unavailable' }, + }); + }, + ); + it('gates a persisted vector-index stamp before filesystem work when the new job is pending', async () => { const f = await fixture(); const previous: NonNullable = { diff --git a/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts b/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts new file mode 100644 index 000000000..ed23c8296 --- /dev/null +++ b/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts @@ -0,0 +1,30 @@ +import { spawn } from 'node:child_process'; +import { expect, it } from 'vitest'; + +it('exits the standalone model process when its owner disconnects', async () => { + const entry = new URL('../../dist/core/embeddings/embedding-sidecar.js', import.meta.url).href; + const child = spawn( + process.execPath, + ['--input-type=module', '-e', `await import(${JSON.stringify(entry)}); process.send('ready');`], + { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + }, + ); + let stderr = ''; + child.stderr?.on('data', (data) => { + stderr += String(data); + }); + const exit = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', resolve); + }); + const ready = new Promise((resolve) => child.once('message', resolve)); + try { + expect(await Promise.race([ready, exit])).toBe('ready'); + child.disconnect(); + expect(await exit, stderr).toBe(1); + } finally { + if (child.connected) child.disconnect(); + await exit; + } +}); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts index 27256e370..2fb09ced7 100644 --- a/gitnexus/test/unit/embedding-streaming-text.test.ts +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -117,8 +117,12 @@ describe('bounded embedding text preserves existing vectors', () => { expect(createHash('sha256').update(JSON.stringify(inputs)).digest('hex')).toBe(digest); expect(release).toHaveBeenCalledTimes(contentRetention === 'full' ? 1 : 0); expect(node).not.toHaveProperty('classDeclaration'); - if (label === 'Class') - expect(inputs.map((input) => input.text).join('\n')).not.toContain('BODY_EXCLUDED'); + expect( + inputs + .filter(() => label === 'Class') + .map((input) => input.text) + .join('\n'), + ).not.toContain('BODY_EXCLUDED'); }, ); @@ -181,7 +185,7 @@ describe('bounded embedding text preserves existing vectors', () => { expect(chunk.endLine).toBe( 13 + content.slice(0, chunk.endOffset - 1).split('\n').length - 1, ); - if (overlap >= 0) expect(chunk.startOffset).toBeLessThanOrEqual(covered); + expect(chunk.startOffset).toBeLessThanOrEqual(covered); covered = chunk.endOffset; } expect(covered).toBe(content.length); diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 14e447485..dbfdc9ef5 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -101,7 +101,8 @@ vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ const IDENTITY = { model: 'test-model', dimensions: 768, provider: 'local' } as const; -vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), effectiveRamBytes: () => 8 * 1024 ** 3, })); vi.mock('node:v8', async (importOriginal) => { @@ -211,12 +212,12 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); - it('keeps automatic reservations within host memory despite a larger default V8 limit', async () => { + it('uses the RAM-sized automatic heap despite a larger default V8 limit', async () => { await store(); await run(); expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( expect.objectContaining({ - resources: expect.objectContaining({ totalBytes: 6 * 1024 ** 3 }), + resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), }), ); }); diff --git a/gitnexus/test/unit/http-embedding-response-budget.test.ts b/gitnexus/test/unit/http-embedding-response-budget.test.ts index e6f2c87a7..5d2d6a994 100644 --- a/gitnexus/test/unit/http-embedding-response-budget.test.ts +++ b/gitnexus/test/unit/http-embedding-response-budget.test.ts @@ -80,6 +80,19 @@ describe('bounded HTTP embedding responses', () => { expect(cancel).toHaveBeenCalledOnce(); }); + it('does not retry a deterministic response-budget violation', async () => { + const embed = await client(); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '3'); + const fetch = vi.fn( + async () => new Response('x'.repeat(100), { headers: { 'content-length': '100' } }), + ); + vi.stubGlobal('fetch', fetch); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(fetch).toHaveBeenCalledOnce(); + }); + it('retains the existing body retry behavior under the byte limit', async () => { const embed = await client(); vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '2'); diff --git a/gitnexus/test/unit/index-lock.test.ts b/gitnexus/test/unit/index-lock.test.ts index 120e09727..8e46b2b0d 100644 --- a/gitnexus/test/unit/index-lock.test.ts +++ b/gitnexus/test/unit/index-lock.test.ts @@ -179,6 +179,18 @@ describe('release', () => { }); describe('sweepStagingArtifacts', () => { + it('reclaims private generation stages and WALs while retaining published graphs', () => { + const generation = `lbug.generation.${'a'.repeat(32)}`; + const stage = `${generation}.staging.6e34c761-bf58-46cc-8b54-78d11607bc46`; + for (const name of [generation, `${generation}.wal`, stage, `${stage}.wal`]) + writeFileSync(path.join(dir, name), 'graph'); + sweepStagingArtifacts(dir); + expect(existsSync(path.join(dir, generation))).toBe(true); + expect(existsSync(path.join(dir, `${generation}.wal`))).toBe(true); + expect(existsSync(path.join(dir, stage))).toBe(false); + expect(existsSync(path.join(dir, `${stage}.wal`))).toBe(false); + }); + const recoveryStage = 'lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46'; const seedRecovery = (stagingFile = recoveryStage): void => { writeFileSync( diff --git a/gitnexus/test/unit/lbug-resource-release.test.ts b/gitnexus/test/unit/lbug-resource-release.test.ts index 7f01d26c2..094e3ed96 100644 --- a/gitnexus/test/unit/lbug-resource-release.test.ts +++ b/gitnexus/test/unit/lbug-resource-release.test.ts @@ -29,7 +29,7 @@ describe('reserved native resource release', () => { await expect(native.closeLbugConnection(h.handle, { strict: true })).rejects.toThrow( 'release failed', ); - if (part === 'conn') expect(h.db.close).not.toHaveBeenCalled(); + expect(h.db.close).toHaveBeenCalledTimes(part === 'conn' ? 0 : 1); await native.closeLbugConnection(h.handle, { strict: true }); expect(h[part].close).toHaveBeenCalledTimes(2); expect(h[part === 'conn' ? 'db' : 'conn'].close).toHaveBeenCalledTimes(1); diff --git a/gitnexus/test/unit/local-backend-vector-generation.test.ts b/gitnexus/test/unit/local-backend-vector-generation.test.ts index 4025cf8de..e57d95da9 100644 --- a/gitnexus/test/unit/local-backend-vector-generation.test.ts +++ b/gitnexus/test/unit/local-backend-vector-generation.test.ts @@ -32,6 +32,7 @@ interface Reader { repo: { lbugPath: string; generationUnknown?: boolean }, query: string, limit: number, + degraded?: { reason?: string }, ): Promise; lastObservedPoolState: Map; lastQueryEmbeddingDims: Map; @@ -44,6 +45,13 @@ beforeEach(() => { }); describe('MCP generation gate', () => { + it('explains the temporary keyword-only lane while vectors are pending', async () => { + const backend = new LocalBackend() as unknown as Reader; + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch({ lbugPath: graphPath }, 'hello', 5, degraded)).toEqual([]); + expect(degraded.reason).toContain('embeddings are pending'); + expect(mocks.embed).not.toHaveBeenCalled(); + }); it.each(['before-read', 'during-read'])( 'clears a prior query width when ownership is lost %s', async (when) => { diff --git a/gitnexus/test/unit/parse-failure-drain.test.ts b/gitnexus/test/unit/parse-failure-drain.test.ts new file mode 100644 index 000000000..f5c368294 --- /dev/null +++ b/gitnexus/test/unit/parse-failure-drain.test.ts @@ -0,0 +1,47 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, it, vi } from 'vitest'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; + +const mocks = vi.hoisted(() => ({ + dispatch: vi.fn(async () => { + throw undefined; + }), + terminate: vi.fn(async () => { + throw new Error('drain failed'); + }), +})); +vi.mock('../../src/core/ingestion/workers/worker-pool.js', async (original) => ({ + ...(await original()), + createWorkerPool: () => ({ + size: 1, + dispatch: mocks.dispatch, + dispatchGroups: mocks.dispatch, + terminate: mocks.terminate, + }), +})); + +it('preserves an undefined parse rejection alongside a native drain failure', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-parse-drain-')); + const source = 'export function example() { return 1; }'; + try { + await fs.writeFile(path.join(root, 'a.ts'), source); + const run = runChunkedParseAndResolve( + createKnowledgeGraph(), + [{ path: 'a.ts', size: source.length }], + ['a.ts'], + 1, + root, + Date.now(), + () => {}, + { workerPoolSize: 1 }, + ); + await expect(run).rejects.toMatchObject({ errors: [undefined, new Error('drain failed')] }); + await expect(run).rejects.toThrow('undefined'); + expect(mocks.terminate).toHaveBeenCalledOnce(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); diff --git a/gitnexus/test/unit/pipeline-runner.test.ts b/gitnexus/test/unit/pipeline-runner.test.ts index c29fdd7f7..4cd953bd4 100644 --- a/gitnexus/test/unit/pipeline-runner.test.ts +++ b/gitnexus/test/unit/pipeline-runner.test.ts @@ -18,6 +18,31 @@ function makeCtx(): PipelineContext { } describe('runPipeline', () => { + it('honors cancellation between native phases without entering the next phase', async () => { + const controller = new AbortController(); + const ctx: PipelineContext = { ...makeCtx(), options: { signal: controller.signal } }; + const entered: string[] = []; + const phases: PipelinePhase[] = [ + { + name: 'parse', + deps: [], + async execute() { + entered.push('parse'); + controller.abort(new Error('cancelled after parse')); + }, + }, + { + name: 'emit', + deps: ['parse'], + async execute() { + entered.push('emit'); + }, + }, + ]; + await expect(runPipeline(phases, ctx)).rejects.toThrow('cancelled after parse'); + expect(entered).toEqual(['parse']); + }); + it('executes phases in dependency order', async () => { const order: string[] = []; diff --git a/gitnexus/test/unit/watch-cancellation.test.ts b/gitnexus/test/unit/watch-cancellation.test.ts index ec6f98486..a0a390034 100644 --- a/gitnexus/test/unit/watch-cancellation.test.ts +++ b/gitnexus/test/unit/watch-cancellation.test.ts @@ -5,7 +5,10 @@ import { execFileSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ analyze: vi.fn(), mayEmbed: vi.fn(async () => false) })); -vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: vi.fn(async () => false) })); +vi.mock('../../src/cli/analyze.js', () => ({ + ensureHeap: vi.fn(async () => false), + usesManualAnalysisHeap: () => false, +})); vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: mocks.analyze, analyzeFailureMayHaveMutatedLiveIndex: () => false, diff --git a/gitnexus/test/unit/watch-external-config.test.ts b/gitnexus/test/unit/watch-external-config.test.ts index 8a4466d20..7808c784b 100644 --- a/gitnexus/test/unit/watch-external-config.test.ts +++ b/gitnexus/test/unit/watch-external-config.test.ts @@ -39,7 +39,10 @@ vi.mock('../../src/core/run-analyze.js', () => ({ return { alreadyUpToDate: true }; }), })); -vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: async () => false })); +vi.mock('../../src/cli/analyze.js', () => ({ + ensureHeap: async () => false, + usesManualAnalysisHeap: () => false, +})); vi.mock('../../src/storage/storage-resolver.js', async (importActual) => ({ ...(await importActual()), requireStoragePath: async () => undefined, diff --git a/gitnexus/test/unit/watch-supervisor-reservation.test.ts b/gitnexus/test/unit/watch-supervisor-reservation.test.ts index 71abd5f9c..e2fa2f0b1 100644 --- a/gitnexus/test/unit/watch-supervisor-reservation.test.ts +++ b/gitnexus/test/unit/watch-supervisor-reservation.test.ts @@ -1,7 +1,8 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, expect, it, vi } from 'vitest'; +import { getHeapStatistics } from 'node:v8'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ resources: vi.fn((_request: { supervisorBytes?: number }) => ({ @@ -10,7 +11,16 @@ const mocks = vi.hoisted(() => ({ })), heap: vi.fn(async () => true), })); -vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: mocks.heap })); +vi.mock('../../src/cli/analyze.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ensureHeap: mocks.heap, usesManualAnalysisHeap: actual.usesManualAnalysisHeap }; +}); +vi.mock('../../src/storage/storage-resolver.js', () => ({ + ANALYZE_STORAGE_REQUIREMENTS: {}, + requireStoragePath: vi.fn(async () => { + throw new Error('stop after admission'); + }), +})); vi.mock('../../src/core/analysis-host-resources.js', () => ({ analysisMayEmbed: async () => false, analysisSupervisorReservation: () => 192 * 1024 ** 2, @@ -23,7 +33,13 @@ import { import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; const directories: string[] = []; +beforeEach(() => { + vi.stubEnv('GITNEXUS_MEMORY', ''); + vi.stubEnv('GITNEXUS_ANALYSIS_MANUAL_HEAP', ''); + vi.stubEnv('NODE_OPTIONS', ''); +}); afterEach(async () => { + process.exitCode = undefined; vi.unstubAllEnvs(); vi.clearAllMocks(); await Promise.all( @@ -60,3 +76,17 @@ it('retains the parent reservation across watch refreshes', async () => { for (const [request] of mocks.resources.mock.calls) expect(request).toMatchObject({ supervisorBytes: 320 * 1024 ** 2 }); }); + +it('honors GITNEXUS_MEMORY=off in the actual watch admission path', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-manual-')); + directories.push(directory); + await fs.mkdir(path.join(directory, '.git')); + vi.stubEnv('GITNEXUS_MEMORY', 'off'); + await watchCommandWithRunnerIdentity({} as AnalyzerRunnerIdentity, directory); + expect(mocks.heap).not.toHaveBeenCalled(); + expect(mocks.resources).toHaveBeenCalledWith( + expect.objectContaining({ + mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + }), + ); +}); diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index 410077b25..3ef56c54a 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -221,6 +221,61 @@ describe('worker pool timeout retirement', () => { } }); + it('replaces a released slot while another slot still awaits physical exit', async () => { + const release: Array<() => void> = []; + const dispatched = new Set(); + const pool = createWorkerPool(workerUrl, 2, { + requireDrain: true, + shutdownDrainMs: 500, + subBatchIdleTimeoutMs: 2000, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id < 2) { + worker.postMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'sub-batch') dispatched.add(worker.id); + }; + worker.terminate = () => + new Promise((resolve) => { + release[worker.id] = () => { + worker.emit('exit', 1); + resolve(1); + }; + }); + } + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + const result = pool.dispatch([ + { path: 'a.ts', content: '' }, + { path: 'b.ts', content: '' }, + ]); + try { + await waitFor(() => dispatched.size === 2, 'Both slots did not dispatch'); + TimeoutThenHealthyWorker.instances[0].emit('error', new Error('slot zero failed')); + TimeoutThenHealthyWorker.instances[1].emit('error', new Error('slot one failed')); + await waitFor(() => release.filter(Boolean).length === 2, 'Missing termination requests'); + release[0](); + await waitFor( + () => TimeoutThenHealthyWorker.instances.length === 3, + 'Released slot was blocked by the other slot', + ); + release[1](); + await result; // Failed inputs are quarantined; both replacement slots remain usable. + expect(TimeoutThenHealthyWorker.instances).toHaveLength(4); + const results = await pool.dispatch([ + { path: 'healthy-a.ts', content: '' }, + { path: 'healthy-b.ts', content: '' }, + ]); + expect(results).toEqual( + expect.arrayContaining([{ paths: ['healthy-a.ts'] }, { paths: ['healthy-b.ts'] }]), + ); + } finally { + for (const finish of release) finish?.(); + await result.catch(() => undefined); + await pool.terminate(); + } + }); + it('does not immediately terminate a worker that timed out inside native parsing', async () => { const pool = createWorkerPool(workerUrl, 1, { subBatchIdleTimeoutMs: 20, diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 08085f6f9..45f3edac1 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -5,6 +5,16 @@ import PerfSequencer from './test/helpers/perf-sequencer.js'; const EMBEDDING_VECTOR_STORE_TEST = 'test/integration/embedding-vector-store.test.ts'; const LEGACY_VECTOR_REUSE_TEST = 'test/integration/legacy-vector-reuse-index.test.ts'; const LEGACY_VECTOR_EXPORT_TEST = 'test/integration/legacy-vector-export.test.ts'; +const RESUMABLE_ANALYSIS_NATIVE_TESTS = [ + 'test/integration/analysis-embedding-flow.test.ts', + 'test/integration/analysis-generation-routing.test.ts', + 'test/integration/analysis-publication-flow.test.ts', + 'test/integration/embedding-job-controller.test.ts', + 'test/integration/embedding-job-restart.test.ts', + 'test/integration/embedding-native-release.test.ts', + 'test/integration/legacy-embedding-controller.test.ts', + 'test/integration/legacy-vector-source.test.ts', +]; export default defineConfig({ test: { @@ -86,6 +96,7 @@ export default defineConfig({ EMBEDDING_VECTOR_STORE_TEST, LEGACY_VECTOR_REUSE_TEST, LEGACY_VECTOR_EXPORT_TEST, + ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', @@ -207,6 +218,7 @@ export default defineConfig({ EMBEDDING_VECTOR_STORE_TEST, LEGACY_VECTOR_REUSE_TEST, LEGACY_VECTOR_EXPORT_TEST, + ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', From 0b3e26d20cd61cd7303bdc7b2c4b4a830960e7ea Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 15:22:28 +0000 Subject: [PATCH 44/67] fix(embeddings): normalize graph parent paths Bind the canonical job root to the same graph path on macOS temp aliases and Windows short paths. Keep graph identity and owned-directory checks. Canonicalize the retention fixture and show subprocess stderr on failure. Validation: 65 focused tests; 26 tests through an aliased TMPDIR; build, test typecheck, formatting and lint; graph change analysis (4 files). --- gitnexus/src/core/embeddings/embedding-job.ts | 12 ++++++++--- .../embedding-job-controller.test.ts | 20 +++++++++++++++++++ .../integration/embedding-job-restart.test.ts | 8 +++++--- .../test/unit/shared-vector-artifact.test.ts | 2 +- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 35a90811e..81c66b68f 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -127,6 +127,12 @@ export async function runPublishedEmbeddingJob( await drainAnalysisNativeResources(); signal?.throwIfAborted(); const root = await embeddingJobDirectory(options.metadataDirectory); + // Canonicalize the parent just as we do for the job root. macOS temporary + // paths can name the same directory through /var and /private/var. + const graphPath = path.join( + await embeddingJobDirectory(path.dirname(path.resolve(options.graphPath))), + path.basename(options.graphPath), + ); let metadata = await options.metadata.loadMetadata(); const graph = metadata?.graphGeneration; const runner = metadata?.runnerIdentity; @@ -142,7 +148,7 @@ export async function runPublishedEmbeddingJob( !metadata.schemaFingerprint || metadata.incrementalInProgress || metadata.graphWriteCollapsed || - !metadataMatchesGraph(root, metadata, options.graphPath) + !metadataMatchesGraph(root, metadata, graphPath) ) throw new Error('Embedding job requires a verified published graph generation'); const config = resolveEmbeddingConfig(options.config); @@ -261,7 +267,7 @@ export async function runPublishedEmbeddingJob( options.log?.(incompatible); legacy = undefined; } else { - const current = await fs.lstat(options.graphPath, { bigint: true }); + const current = await fs.lstat(graphPath, { bigint: true }); if (String(current.dev) === legacy.file.dev && String(current.ino) === legacy.file.ino) throw new Error('Legacy reuse requires a retired source distinct from the mutable graph'); } @@ -345,7 +351,7 @@ export async function runPublishedEmbeddingJob( if (!manifestExists) await clearUnstartedManifest(directory); const getSource = async (): Promise => (source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)( - options.graphPath, + graphPath, resources.graphPoolBytes, resources.nativeThreads, )); diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index eb02c2022..9a817172a 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -810,6 +810,26 @@ describe('published graph embedding job controller', () => { await expect(fs.stat(path.join(job, 'vectors'))).rejects.toMatchObject({ code: 'ENOENT' }); }); + it('binds graph paths through an aliased ancestor to the canonical job root', async () => { + const f = await fixture(1); + const alias = path.join(f.root, 'alias'); + await fs.symlink( + path.dirname(f.root), + alias, + process.platform === 'win32' ? 'junction' : 'dir', + ); + const graphFile = path.basename(f.options().graphPath); + const result = await f.run({ + graphPath: path.join(alias, path.basename(f.root), graphFile), + }); + expect(result.phase).toBe('ready'); + expect(f.openSource).toHaveBeenCalledWith( + path.join(await fs.realpath(f.root), graphFile), + f.options().resources.graphPoolBytes, + f.options().resources.nativeThreads, + ); + }); + it('refuses a symlink job root before writing metadata or requesting vectors', async () => { const f = await fixture(5); await fs.mkdir(path.join(f.root, 'other')); diff --git a/gitnexus/test/integration/embedding-job-restart.test.ts b/gitnexus/test/integration/embedding-job-restart.test.ts index 4e4367769..6086a6d7d 100644 --- a/gitnexus/test/integration/embedding-job-restart.test.ts +++ b/gitnexus/test/integration/embedding-job-restart.test.ts @@ -72,9 +72,11 @@ it.each(boundaries)( }, }, }); - await expect( - execute(process.execPath, [helper, root, boundary], { env }), - ).rejects.toMatchObject({ code: 73 }); + const stopped = await execute(process.execPath, [helper, root, boundary], { env }).then( + () => ({ code: 0, stderr: '' }), + (error: NodeJS.ErrnoException & { stderr?: string }) => error, + ); + expect(stopped.code, stopped.stderr).toBe(73); await execute(process.execPath, [helper, root, 'resume'], { env }); const readInputs = async ( phase: string, diff --git a/gitnexus/test/unit/shared-vector-artifact.test.ts b/gitnexus/test/unit/shared-vector-artifact.test.ts index 014a8772e..df0399dd1 100644 --- a/gitnexus/test/unit/shared-vector-artifact.test.ts +++ b/gitnexus/test/unit/shared-vector-artifact.test.ts @@ -25,7 +25,7 @@ let root: string, preparing: string; const differentDevice = () => Object.assign(new Error('Different filesystem'), { code: 'EXDEV' }); beforeEach(async () => { - root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-retain-vectors-')); + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-retain-vectors-'))); source = path.join(root, 'source'); destination = path.join(root, 'destination'); original = path.join(source, 'embeddings', generation.identity, 'vectors'); From a21af76036aeaa3563872575d647587612af0b6d Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 15:36:52 +0000 Subject: [PATCH 45/67] fix(embeddings): batch writes and guard recovery Batch bounded vector and receipt writes to remove per-node native calls without changing validation, transactions, or durable checkpoints. Pin publication descriptors before entry validation. Preserve excluded vector coverage, reject malformed status paths, retry orphan sidecars, and complete shutdown after child spawn failures. Validation: 159 focused tests and 89 native/restart tests; build, test typecheck, lint and formatting. Refreshed graph analysis maps production changes; three test files remain unmapped. --- gitnexus/src/core/analysis-publication.ts | 13 +++-- gitnexus/src/core/lbug/vector-store.ts | 50 +++++++++++-------- gitnexus/src/server/analyze-job.ts | 14 +++++- .../src/storage/analysis-artifact-cleanup.ts | 15 ++++-- .../src/storage/embedding-job-progress.ts | 2 + gitnexus/src/storage/vector-generation.ts | 2 +- .../embedding-vector-store.test.ts | 35 +++++++++++++ .../test/unit/analysis-publication.test.ts | 39 +++++++++++++++ gitnexus/test/unit/analyze-job.test.ts | 16 ++++++ .../test/unit/embedding-job-progress.test.ts | 10 ++++ gitnexus/test/unit/vector-generation.test.ts | 16 ++++++ 11 files changed, 177 insertions(+), 35 deletions(-) diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts index 908f2e61c..e2bf88146 100644 --- a/gitnexus/src/core/analysis-publication.ts +++ b/gitnexus/src/core/analysis-publication.ts @@ -106,13 +106,16 @@ async function regularFile(file: string): Promise { } async function boundedRead(file: string, maximum: number): Promise { - const before = await fs.lstat(file); - if (!before.isFile() || before.isSymbolicLink()) - throw new Error('Publication artifact must be an owned regular file'); - const handle = await fs.open(file, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const handle = await fs.open( + file, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); try { const stat = await handle.stat(); - if (stat.dev !== before.dev || stat.ino !== before.ino) + const entry = await fs.lstat(file); + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Publication artifact must be an owned regular file'); + if (stat.dev !== entry.dev || stat.ino !== entry.ino) throw new Error('Publication artifact changed while opening'); if (!stat.isFile() || stat.size > maximum) throw new Error('Publication artifact exceeds its receipt bound'); diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 3d1c894af..28dceb62f 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -479,18 +479,15 @@ export class VectorStore { await remove(); } await this.transaction(async () => { - for (const { nodeId, contentHash, sourceStartLine = 0, sourceEndLine = 0 } of nodes) { - if (previous.has(nodeId)) - await this.query( - 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.chunks=0, n.sourceStartLine=$sourceStartLine, n.sourceEndLine=$sourceEndLine', - { nodeId, contentHash, sourceStartLine, sourceEndLine }, - ); - else - await this.query( - 'CREATE (:EmbeddingNodeReceipt {nodeId:$nodeId, contentHash:$contentHash, chunks:0, complete:false, sourceStartLine:$sourceStartLine, sourceEndLine:$sourceEndLine})', - { nodeId, contentHash, sourceStartLine, sourceEndLine }, - ); - } + await this.query( + 'UNWIND range(1, size($ids)) AS i MERGE (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.chunks=0, n.complete=false, n.sourceStartLine=$firstLines[i], n.sourceEndLine=$lastLines[i]', + { + ids: nodes.map((node) => node.nodeId), + hashes: nodes.map((node) => node.contentHash), + firstLines: nodes.map((node) => node.sourceStartLine ?? 0), + lastLines: nodes.map((node) => node.sourceEndLine ?? 0), + }, + ); }); }, true); } @@ -546,16 +543,25 @@ export class VectorStore { receipt.chunks++; } await this.transaction(async () => { - for (const chunk of chunks) - await this.query( - 'CREATE (:CodeEmbedding {id:$id, nodeId:$nodeId, chunkIndex:$chunkIndex, startLine:$startLine, endLine:$endLine, embedding:$embedding, contentHash:$contentHash})', - { id: `${chunk.nodeId}:${chunk.chunkIndex}`, ...chunk }, - ); - for (const receipt of receipts.values()) - await this.query('MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.chunks=$chunks', { - nodeId: receipt.nodeId, - chunks: receipt.chunks, - }); + await this.query( + 'UNWIND range(1, size($ids)) AS i CREATE (:CodeEmbedding {id:$ids[i], nodeId:$nodeIds[i], chunkIndex:$ordinals[i], startLine:$firstLines[i], endLine:$lastLines[i], embedding:$vectors[i], contentHash:$hashes[i]})', + { + ids: chunks.map((chunk) => `${chunk.nodeId}:${chunk.chunkIndex}`), + nodeIds: chunks.map((chunk) => chunk.nodeId), + ordinals: chunks.map((chunk) => chunk.chunkIndex), + firstLines: chunks.map((chunk) => chunk.startLine), + lastLines: chunks.map((chunk) => chunk.endLine), + vectors: chunks.map((chunk) => chunk.embedding), + hashes: chunks.map((chunk) => chunk.contentHash), + }, + ); + await this.query( + 'UNWIND range(1, size($ids)) AS i MATCH (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.chunks=$chunks[i]', + { + ids: [...receipts.keys()], + chunks: [...receipts.values()].map((receipt) => receipt.chunks), + }, + ); }); }, true); } diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index b82727509..73fd1c3aa 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -384,8 +384,18 @@ export class JobManager { [...this.children.values()].map( (child) => new Promise((resolve) => { - if (child.exitCode !== null || child.signalCode !== null) resolve(); - else child.once('exit', () => resolve()); + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + const finished = () => { + child.removeListener('exit', finished); + child.removeListener('close', finished); + resolve(); + }; + child.once('exit', finished); + // Spawn failures emit error and close, but never exit. + child.once('close', finished); }), ), ); diff --git a/gitnexus/src/storage/analysis-artifact-cleanup.ts b/gitnexus/src/storage/analysis-artifact-cleanup.ts index 4742eae7b..b32202316 100644 --- a/gitnexus/src/storage/analysis-artifact-cleanup.ts +++ b/gitnexus/src/storage/analysis-artifact-cleanup.ts @@ -92,11 +92,16 @@ export async function reclaimAnalysisArtifacts( return false; } }; - for (const entry of entries) { - if (!entry.isFile() || !GRAPH.test(entry.name) || graphs.has(entry.name)) continue; - if (await remove(path.join(root, entry.name))) - for (const suffix of ['.wal', '.shadow']) - await remove(path.join(root, entry.name + suffix)); + // A busy sidecar may outlive its base graph; include it in the next sweep. + const candidates = new Set( + entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name.replace(/\.(wal|shadow)$/, '')), + ); + for (const graph of candidates) { + if (!GRAPH.test(graph) || graphs.has(graph)) continue; + if (await remove(path.join(root, graph))) + for (const suffix of ['.wal', '.shadow']) await remove(path.join(root, graph + suffix)); } for (const job of orphanJobs) { // Keep the ownership receipt until every payload is gone. Recursive diff --git a/gitnexus/src/storage/embedding-job-progress.ts b/gitnexus/src/storage/embedding-job-progress.ts index dee2e2345..0716c0444 100644 --- a/gitnexus/src/storage/embedding-job-progress.ts +++ b/gitnexus/src/storage/embedding-job-progress.ts @@ -21,6 +21,8 @@ export async function readEmbeddingJobProgress( !metadata?.vectorGeneration?.excludedNodes ) return undefined; + if (metadata.graphPath !== undefined && typeof metadata.graphPath !== 'string') + return { phase: 'unverified' }; const selected = selectVectorGeneration( metadataDirectory, metadata, diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index a671afd52..b66088a2f 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -107,7 +107,7 @@ export function selectVectorGeneration( vector.excludedNodes > vector.nodes || vector.exclusionReason !== 'input-rejected')) || !nonnegative(vector.chunks) || - vector.chunks < vector.nodes || + vector.chunks < vector.nodes - (vector.excludedNodes ?? 0) || (vector.indexKind !== 'hnsw' && vector.indexKind !== 'exact') ) return { kind: 'unavailable', reason: 'vector-generation-mismatch' }; diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index 5d473c95a..f3808ca7f 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -824,6 +824,41 @@ describe('separate durable vector artifacts', () => { expect(recovered.contentHash).toBe(next.contentHash); expect(await store.readChunks(recovered, 0, 1)).toEqual(original); }); + it('preserves each node, chunk ordinal, line range and vector in a mixed batch', async () => { + const options = await setup(); + const nodes = [0, 1].map((i) => ({ + nodeId: `Function:a:mixed${i}`, + contentHash: `hash${i}`, + sourceStartLine: i * 10, + sourceEndLine: i * 10 + 5, + })); + const rows = [ + { ...nodes[0], chunkIndex: 0, startLine: 0, endLine: 2, embedding: [1, 2, 3, 4] }, + { ...nodes[1], chunkIndex: 0, startLine: 10, endLine: 15, embedding: [5, 6, 7, 8] }, + { ...nodes[0], chunkIndex: 1, startLine: 3, endLine: 5, embedding: [9, 10, 11, 12] }, + ]; + await store.beginNodes(nodes); + await store.appendBatch(rows); + const receipts = await store.finishNodes( + nodes.map((node, i) => ({ ...node, chunks: i === 0 ? 2 : 1 })), + ); + await store.checkpointUnit(receipts); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + for (const receipt of receipts) { + expect(await store.readChunks(receipt, 0, receipt.chunks)).toEqual( + rows + .filter((row) => row.nodeId === receipt.nodeId) + .map(({ chunkIndex, startLine, endLine, embedding }) => ({ + chunkIndex, + startLine, + endLine, + embedding, + })), + ); + } + }); + it('verifies a complete unit with bounded native queries instead of one query per node', async () => { await setup(); const nodes = Array.from({ length: 64 }, (_, i) => ({ diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts index b24b83340..273882571 100644 --- a/gitnexus/test/unit/analysis-publication.test.ts +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -224,6 +224,45 @@ describe('durable graph publication', () => { await expect(fs.stat(graph)).rejects.toMatchObject({ code: 'ENOENT' }); }); + it.each(['.wal', '.shadow'])( + 'retries a busy orphan %s after deleting its graph', + async (suffix) => { + const { root } = await fixture(); + const graph = path.join(root, `lbug.generation.${'b'.repeat(32)}`); + await fs.writeFile(graph, 'retired graph'); + await fs.writeFile(graph + suffix, 'busy sidecar'); + const remove = fs.rm.bind(fs); + const blocked = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === graph + suffix) + throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + await remove(file, options); + }); + await reclaimAnalysisArtifacts(root, metadata, undefined, vi.fn()); + await expect(fs.stat(graph)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(graph + suffix, 'utf8')).toBe('busy sidecar'); + blocked.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata); + await expect(fs.stat(graph + suffix)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('rejects a publication receipt replaced after its descriptor is opened', async () => { + const { root, stage } = await fixture(); + await prepareAnalysisPublication(root, stage, metadata, content); + const receipt = path.join(root, 'analysis-publication.json'); + const open = fs.open.bind(fs); + vi.spyOn(fs, 'open').mockImplementation(async (...args) => { + const handle = await open(...args); + if (String(args[0]) === receipt) { + await fs.rename(receipt, receipt + '.retained'); + await fs.writeFile(receipt, 'foreign replacement'); + } + return handle; + }); + await expect(readPreparedAnalysis(root)).rejects.toThrow('changed while opening'); + expect(await fs.readFile(receipt, 'utf8')).toBe('foreign replacement'); + }); + it('recovers a prior process incarnation even when its PID is now this process', async () => { const { root, stage, actions } = await fixture(); const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { diff --git a/gitnexus/test/unit/analyze-job.test.ts b/gitnexus/test/unit/analyze-job.test.ts index 37cba5636..98e74f3aa 100644 --- a/gitnexus/test/unit/analyze-job.test.ts +++ b/gitnexus/test/unit/analyze-job.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { EventEmitter } from 'node:events'; +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; import { JobManager, isTerminalJobStatus, @@ -429,6 +433,18 @@ describe('JobManager', () => { expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); }); + it('finishes shutdown when a registered child fails to spawn without emitting exit', async () => { + const job = manager.createJob({ repoPath: '/tmp/repo' }); + const child = spawn(process.execPath, [], { cwd: path.join(os.tmpdir(), randomUUID()) }); + child.on('error', () => manager.releaseChild(job.id)); + const exit = vi.fn(); + child.on('exit', exit); + manager.registerChild(job.id, child); + await manager.waitForChildren(); + expect(exit).not.toHaveBeenCalled(); + expect(child.listenerCount('close')).toBe(0); + }); + it('cancelJob leaves a disconnected native worker running until it exits', () => { const job = manager.createJob({ repoPath: '/tmp/repo' }); manager.updateJob(job.id, { status: 'analyzing' }); diff --git a/gitnexus/test/unit/embedding-job-progress.test.ts b/gitnexus/test/unit/embedding-job-progress.test.ts index 865593753..b86fc5169 100644 --- a/gitnexus/test/unit/embedding-job-progress.test.ts +++ b/gitnexus/test/unit/embedding-job-progress.test.ts @@ -52,6 +52,16 @@ afterEach(async () => { }); describe('bounded durable embedding progress', () => { + it.each([42, {}, null])('reports malformed graph paths as unverified (%j)', async (graphPath) => { + const metadata = { + ...meta, + embeddingJob: undefined, + graphPath, + vectorGeneration: { preservedForGraph: generation }, + } as unknown as RepoMeta; + expect(await readEmbeddingJobProgress('/missing', metadata)).toEqual({ phase: 'unverified' }); + }); + it.each(['interrupted', 'collapsed'] as const)( 'rejects a legacy vector capability after an %s graph write', async (state) => { diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts index 24f2902d0..6f6ec092c 100644 --- a/gitnexus/test/unit/vector-generation.test.ts +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -128,6 +128,22 @@ describe('vector generation routing', () => { }), ).toMatchObject({ kind: 'unavailable' }); }); + it('admits completed coverage with excluded inputs but rejects missing healthy chunks', () => { + const vectorGeneration = { + ...meta.vectorGeneration, + nodes: 3, + chunks: 2, + excludedNodes: 1, + exclusionReason: 'input-rejected' as const, + }; + expect(select({ ...meta, vectorGeneration })).toMatchObject({ kind: 'ready' }); + expect(select({ ...meta, vectorGeneration: { ...vectorGeneration, chunks: 1 } })).toMatchObject( + { + kind: 'unavailable', + reason: 'vector-generation-mismatch', + }, + ); + }); it.each([NaN, -1, 0.5, Infinity])('rejects invalid coverage %s', (chunks) => { expect( select({ ...meta, vectorGeneration: { ...meta.vectorGeneration, chunks } }), From 01f8600c74b197e128e54251fd2abb80a3e23a87 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 16:01:06 +0000 Subject: [PATCH 46/67] ci: profile fresh analyze overhead Capture process and parser-worker CPU profiles in a separate CI-only run. Reject profiling with --check so diagnostic overhead never affects the acceptance gate. --- .github/workflows/ci-tests.yml | 19 +++++++++++++++++++ gitnexus/bench/analyze-resume/README.md | 5 +++++ gitnexus/bench/analyze-resume/measure.mjs | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 940ad0900..21ff30092 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -49,6 +49,25 @@ jobs: - name: Install candidate search extensions working-directory: gitnexus run: npx tsx scripts/ensure-fts.ts + - name: Profile fresh HTTP analysis separately from acceptance timings + if: matrix.resources == 'explicit' + working-directory: gitnexus + run: | + mkdir -p "$RUNNER_TEMP/analyze-profile" + node bench/analyze-resume/measure.mjs --cpu-profile --allow-dependency-changes \ + --scenarios http-fresh --pairs 1 --warmups 0 \ + --baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \ + --output "$RUNNER_TEMP/analyze-profile/results.json" + - name: Retain diagnostic CPU profiles before the full comparison + if: always() && matrix.resources == 'explicit' + uses: ./.github/actions/upload-test-artifact + with: + name: analyze-resume-cpu-profile + path: | + ${{ runner.temp }}/analyze-profile/**/*.cpuprofile + ${{ runner.temp }}/analyze-profile/**/*.json + ${{ runner.temp }}/analyze-profile/**/*.log + retention-days: 7 - name: Check paired analyze and resume performance working-directory: gitnexus shell: bash diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index 86221d0af..b0c4a122d 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -46,6 +46,11 @@ Each job runs `--validate-gate` decision checks before measuring and uploads raw samples, summaries, dependency receipts and analyzer logs, including partial results on failure. The runner never overwrites an existing report. +The explicit-profile CI job also uploads `analyze-resume-cpu-profile` before the +full comparison. This separate one-pair fresh-HTTP run uses `--cpu-profile` to +capture Node CPU profiles for each process and parser worker. Profiling is +incompatible with `--check`; acceptance timings always run without profiling. + Benchmark execution belongs exclusively to the dedicated CI jobs above. It is not included in the unit or integration test suites. diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs index e8e4dcec5..05591a4cc 100644 --- a/gitnexus/bench/analyze-resume/measure.mjs +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -387,6 +387,7 @@ async function runCase({ heapMb, workers, defaultResources, + cpuProfile, provider, sampleMs, stopRequested, @@ -416,7 +417,13 @@ async function runCase({ GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '16', }); - const args = [cli, 'analyze', repo, '--index-only']; + const args = [ + ...(cpuProfile ? ['--cpu-prof', `--cpu-prof-dir=${directory}`] : []), + cli, + 'analyze', + repo, + '--index-only', + ]; if (!defaultResources) args.push('--memory-budget', String(heapMb), '--workers', String(workers)); if (embedding) args.push('--embeddings', '0'); const counters = provider.begin( @@ -516,6 +523,7 @@ export async function main(argv = process.argv.slice(2)) { 'heap-mb': { type: 'string', default: '2048' }, workers: { type: 'string', default: '2' }, 'default-resources': { type: 'boolean', default: false }, + 'cpu-profile': { type: 'boolean', default: false }, 'allow-dependency-changes': { type: 'boolean', default: false }, 'sample-ms': { type: 'string', default: '25' }, 'keep-stores': { type: 'boolean', default: false }, @@ -524,6 +532,10 @@ export async function main(argv = process.argv.slice(2)) { help: { type: 'boolean' }, }, }); + assert( + !(v.check && v['cpu-profile']), + 'CPU profiles are diagnostic runs, not acceptance timings', + ); if (v['validate-gate']) { const fixture = { schemaVersion: 1, maxPairedRatio: 1, scenarios: { case: {} } }; const summary = (ratios) => @@ -571,6 +583,7 @@ export async function main(argv = process.argv.slice(2)) { --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 --default-resources Omit heap, worker and native-pool overrides on both sides + --cpu-profile Retain per-process/thread CPU profiles; incompatible with --check --allow-dependency-changes Compare shipped runtimes, including intentional dependency changes --validate-gate Check decision logic without running benchmarks --check Require the baseline corpus/scenarios and reject detected slowdowns @@ -653,6 +666,7 @@ Live-query latency and platform cases remain separate acceptance requirements.`) totalMemory: os.totalmem(), settings: numbers, resourceProfile: v['default-resources'] ? 'default' : 'explicit-heap-workers-pool', + cpuProfile: v['cpu-profile'], comparison: v['allow-dependency-changes'] ? 'shipped-runtimes' : 'identical-dependencies', runtime, clis: {}, @@ -660,7 +674,7 @@ Live-query latency and platform cases remain separate acceptance requirements.`) limitations: [ 'Synthetic corpus only; stats and deterministic input coverage are checked, not query ranking.', 'Linux RSS sampling is a lower-bound observation, not a hard memory limit or exact peak.', - 'CPU/disk, phase timings, native reservations and queue peaks are not collected by this harness.', + 'Disk, phase timings, native reservations and queue peaks are not collected; optional CPU profiles are diagnostic only.', 'Only the selected scenarios and resource profile were measured; live query latency and Windows require additional validation.', ], acceptance: 'incomplete', @@ -707,6 +721,7 @@ Live-query latency and platform cases remain separate acceptance requirements.`) heapMb: numbers['heap-mb'], workers: numbers.workers, defaultResources: v['default-resources'], + cpuProfile: v['cpu-profile'], sampleMs: numbers['sample-ms'], provider, stopRequested: () => interrupted, From dec446d82b7450ec3f7283837c52fb0a6e34797f Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 16:24:36 +0000 Subject: [PATCH 47/67] perf(analyze): trim continuation startup work --- .github/workflows/ci-tests.yml | 2 +- gitnexus/src/cli/index.ts | 19 +++-- gitnexus/src/core/run-analyze.ts | 7 +- gitnexus/test/unit/analyze-bootstrap.test.ts | 74 ++++++++++++++++++++ 4 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 gitnexus/test/unit/analyze-bootstrap.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 21ff30092..b5ef48529 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -62,7 +62,7 @@ jobs: if: always() && matrix.resources == 'explicit' uses: ./.github/actions/upload-test-artifact with: - name: analyze-resume-cpu-profile + name: analyze-resume-cpu-profile-${{ matrix.resources }} path: | ${{ runner.temp }}/analyze-profile/**/*.cpuprofile ${{ runner.temp }}/analyze-profile/**/*.json diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 25a398347..0f0479c78 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -290,14 +290,25 @@ program process.env.GITNEXUS_EMBEDDING_DIMS = dimsEnvBaseline; } }) - .action( - createAnalyzerLbugLazyAction( + .action(async (inputPath, options) => { + // The connected continuation only publishes the writer's prepared graph + // and embeds it. Its analyzer receipt is never used: the graph writer has + // already finalized provenance. Keep the native check, without repeating + // the complete analyzer identity snapshot in this separate process. + if (process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected) { + await createLbugLazyAction(() => import('./analyze.js'), 'analyzeCommand')( + inputPath, + options, + ); + return; + } + await createAnalyzerLbugLazyAction( () => import('../core/analyzer-identity.js'), () => import('./analyze.js'), 'analyzeOrWatchCommandWithRunnerIdentity', import.meta.url, - ), - ); + )(inputPath, options); + }); program .command('index [path...]') diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 6b3c3085c..ac65721d5 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -51,7 +51,6 @@ import { summarizeNameFallback, countCallsByLanguage, } from './ingestion/scope-resolution/name-fallback-summary.js'; -import { runPipelineFromRepo } from './ingestion/pipeline.js'; import type { RouteHookConfig } from './ingestion/language-provider.js'; import { routeHookConfigKey } from './ingestion/route-hook-config.js'; import { @@ -1573,7 +1572,10 @@ export async function continuePreparedAnalysis( const resources = options.resources; if (!resources) throw new Error('Graph continuation requires admitted resources'); const log = (message: string) => callbacks.onLog?.(stripControlCharacters(message)); - let target = await resolveWriteTarget(handoff.repoPath, options); + // Publication resolves the target before locking and again under the lock. + // Retain that verified target for its callbacks; an extra snapshot here is + // overwritten before use and adds synchronous Git probes to every handoff. + let target: Awaited>; const resolveTarget = async (): Promise => { target = await resolveWriteTarget(handoff.repoPath, options); return { @@ -3650,6 +3652,7 @@ async function runFullAnalysisInner( ); } callbacks.signal?.throwIfAborted(); + const { runPipelineFromRepo } = await import('./ingestion/pipeline.js'); pipelineResult = await runPipelineFromRepo( repoPath, (p) => { diff --git a/gitnexus/test/unit/analyze-bootstrap.test.ts b/gitnexus/test/unit/analyze-bootstrap.test.ts new file mode 100644 index 000000000..80a816dc2 --- /dev/null +++ b/gitnexus/test/unit/analyze-bootstrap.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + capture: vi.fn(), + check: vi.fn(() => ({ ok: true })), + analyze: vi.fn(), + analyzeOrWatch: vi.fn(), +})); + +vi.mock('../../src/core/analyzer-identity.js', () => ({ + captureAnalyzerIdentityBeforeLoad: mocks.capture, +})); +vi.mock('../../src/core/lbug/native-check.js', () => ({ + prepareLbugNativeBinary: vi.fn(), + checkLbugNative: mocks.check, +})); +vi.mock('../../src/cli/analyze.js', () => ({ + analyzeCommand: mocks.analyze, + analyzeOrWatchCommandWithRunnerIdentity: mocks.analyzeOrWatch, +})); +vi.mock('../../src/cli/update-notice.js', () => ({ runProcessCliUpdateNotice: vi.fn() })); + +const originalArgv = process.argv; +const connected = Object.getOwnPropertyDescriptor(process, 'connected'); +const receipt = { schemaVersion: 4 }; + +beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + mocks.capture.mockImplementation(async (_url, loader) => ({ + runnerIdentity: receipt, + loaded: await loader(), + })); + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + // Vitest's fork also owns IPC. Keep it connected while exercising the CLI + // dispatcher, so its own result channel is not disrupted by this fixture. + Object.defineProperty(process, 'connected', { configurable: true, value: true }); +}); + +afterEach(() => { + process.argv = originalArgv; + if (connected) Object.defineProperty(process, 'connected', connected); + else delete (process as Partial).connected; + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +it.each([ + { continuation: false, watch: false, captures: true }, + { continuation: false, watch: true, captures: true }, + { continuation: true, watch: false, captures: false }, +])('selects the analyzer bootstrap for %j', async (scenario) => { + vi.stubEnv('GITNEXUS_ANALYSIS_CONTINUATION', scenario.continuation ? '1' : ''); + process.argv = [process.execPath, 'gitnexus', 'analyze', '/fixture']; + if (scenario.watch) process.argv.push('--watch'); + + await import('../../src/cli/index.js'); + if (scenario.captures) { + await vi.waitFor(() => expect(mocks.analyzeOrWatch).toHaveBeenCalledOnce()); + expect(mocks.analyzeOrWatch).toHaveBeenCalledWith( + receipt, + '/fixture', + expect.objectContaining(scenario.watch ? { watch: true } : {}), + ); + expect(mocks.capture).toHaveBeenCalledOnce(); + expect(mocks.analyze).not.toHaveBeenCalled(); + } else { + await vi.waitFor(() => expect(mocks.analyze).toHaveBeenCalledOnce()); + expect(mocks.analyze).toHaveBeenCalledWith('/fixture', expect.any(Object)); + expect(mocks.capture).not.toHaveBeenCalled(); + expect(mocks.analyzeOrWatch).not.toHaveBeenCalled(); + } + expect(mocks.check).toHaveBeenCalledOnce(); +}); From 9c18dbaba6e34240a674f1bf62bd1ce645ac48c3 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 16:44:42 +0000 Subject: [PATCH 48/67] fix(analyze): preserve recovery and storage bounds Keep automatic writer pools page-scaled and restore frontend heap caps. Adopt a generation recovered during handoff, reclaim sealed artifacts, and retire compatible legacy sources after migration completes. Report semantic degradation and omit provider causes from API logs. Enforce no paid replay in the CI resume benchmark. --- MIGRATION.md | 15 ++ gitnexus/bench/analyze-resume/README.md | 2 +- gitnexus/bench/analyze-resume/measure.mjs | 13 ++ .../src/core/analysis-graph-verification.ts | 11 +- gitnexus/src/core/analysis-host-resources.ts | 6 +- .../src/core/analysis-phase-supervisor.ts | 39 +++-- gitnexus/src/core/analysis-resource-budget.ts | 10 +- .../core/auto-sync/analysis-worker-launch.ts | 1 + gitnexus/src/core/embeddings/embedding-job.ts | 35 ++++- .../src/core/embeddings/vector-publication.ts | 12 ++ gitnexus/src/core/run-analyze.ts | 8 +- gitnexus/src/mcp/local/local-backend.ts | 8 + gitnexus/src/server/analyze-launch.ts | 6 +- gitnexus/src/server/api.ts | 7 +- .../src/storage/analysis-artifact-cleanup.ts | 38 ++--- gitnexus/src/storage/embedding-job.ts | 1 + .../src/storage/shared-vector-artifact.ts | 5 +- .../embedding-job-controller.test.ts | 17 +++ .../legacy-embedding-controller.test.ts | 3 + .../test/unit/analysis-host-resources.test.ts | 5 +- .../unit/analysis-phase-supervisor.test.ts | 37 ++++- .../test/unit/analysis-publication.test.ts | 141 ++++++++++-------- .../unit/analysis-resource-budget.test.ts | 21 +++ .../test/unit/analyze-launch-collapse.test.ts | 36 +++++ gitnexus/test/unit/api-fts-mode.test.ts | 60 +++++--- .../unit/auto-sync-analysis-worker.test.ts | 40 +++++ .../local-backend-vector-generation.test.ts | 45 +++++- 27 files changed, 488 insertions(+), 134 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 487684443..62481cba2 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -31,6 +31,21 @@ three deterministic input rejections, the job publishes healthy nodes and record On Windows, a legacy reader can prevent retirement of `lbug`. Stop that reader and rerun analyze; the prepared publication and vector artifacts are retained. +`GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` is no longer accepted by analysis: it asks +the native engine to allocate an unbounded pool. Remove this setting from CLI, +server and auto-sync environments to use automatic sizing, or set a positive +byte limit. An explicit `--resource-budget` must also fit the page-size-scaled +COPY floor; increase the budget if admission rejects it. + +If a superseded or unverifiable `analysis-publication.json` blocks recovery, +stop every reader and writer and back up the complete storage slot first. +Move only that receipt into the backup, outside the slot, then run +`gitnexus analyze --force`. Keep the staged graphs and embedding artifacts until +the replacement graph and vectors are verified. A valid receipt should normally +be recovered by plain analysis; discarding it abandons its prepared graph work. +For an interrupted in-place write, keep clients stopped until the forced rebuild +completes; removing the receipt does not make the old graph healthy. + ## LadybugDB 0.22.0 storage and extensions LadybugDB 0.22.0 reads indexes written by 0.21.1, but a write/checkpoint upgrades diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md index b0c4a122d..048030339 100644 --- a/gitnexus/bench/analyze-resume/README.md +++ b/gitnexus/bench/analyze-resume/README.md @@ -46,7 +46,7 @@ Each job runs `--validate-gate` decision checks before measuring and uploads raw samples, summaries, dependency receipts and analyzer logs, including partial results on failure. The runner never overwrites an existing report. -The explicit-profile CI job also uploads `analyze-resume-cpu-profile` before the +The explicit-profile CI job also uploads `analyze-resume-cpu-profile-explicit` before the full comparison. This separate one-pair fresh-HTTP run uses `--cpu-profile` to capture Node CPU profiles for each process and parser worker. Profiling is incompatible with `--check`; acceptance timings always run without profiling. diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs index 05591a4cc..fe0511953 100644 --- a/gitnexus/bench/analyze-resume/measure.mjs +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -748,6 +748,19 @@ Live-query latency and platform cases remain separate acceptance requirements.`) pair.baseline.equalWork, `Unequal work for ${scenario}; inspect retained measurements`, ); + if (scenario === 'http-resume') { + assert.equal( + pair.candidate.providerAfter.acceptedInputs, + pair.candidate.providerAfter.uniqueInputs, + 'Resume replayed already accepted embedding inputs', + ); + assert.equal( + pair.candidate.measuredProviderInputs, + pair.candidate.providerAfter.uniqueInputs - + pair.candidate.providerBefore.acceptedInputs, + 'Resume must request only unfinished embedding inputs', + ); + } const baselineDependencies = pair.baseline.metadata.runnerIdentity?.dependencyRuntime?.digest; assert( diff --git a/gitnexus/src/core/analysis-graph-verification.ts b/gitnexus/src/core/analysis-graph-verification.ts index 895b1ece6..9999e6578 100644 --- a/gitnexus/src/core/analysis-graph-verification.ts +++ b/gitnexus/src/core/analysis-graph-verification.ts @@ -2,7 +2,7 @@ import lbug from '@ladybugdb/core'; import type { RepoMeta } from '../storage/repo-meta.js'; import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_FINGERPRINT } from './lbug/schema.js'; -import { closeLbugConnection, openLbugConnection } from './lbug/lbug-config.js'; +import { closeLbugConnection, openLbugConnection, estimateBufferPool } from './lbug/lbug-config.js'; import { assertLbugPublicationReady } from './lbug/publication.js'; import { getFtsIndexes } from './search/fts-schema.js'; import { verifySearchFTSIndexes } from './search/fts-indexes.js'; @@ -32,7 +32,14 @@ export async function verifyPublishedAnalysisGraph( const expectedNodes = count(metadata.stats?.nodes); const expectedEdges = count(metadata.stats?.edges); await assertLbugPublicationReady(graphPath); - const handle = await openLbugConnection(lbug, graphPath, { ...resources, readOnly: true }); + const handle = await openLbugConnection(lbug, graphPath, { + ...resources, + bufferPoolBytes: Math.min( + resources.bufferPoolBytes, + estimateBufferPool(expectedNodes + expectedEdges), + ), + readOnly: true, + }); const query = async (sql: string): Promise[]> => { const result = await handle.conn.query(sql); const cursors = Array.isArray(result) ? result : [result]; diff --git a/gitnexus/src/core/analysis-host-resources.ts b/gitnexus/src/core/analysis-host-resources.ts index 30fba8255..e4c74a40f 100644 --- a/gitnexus/src/core/analysis-host-resources.ts +++ b/gitnexus/src/core/analysis-host-resources.ts @@ -2,6 +2,7 @@ import os from 'node:os'; import path from 'node:path'; import { effectiveRamBytes } from './ingestion/utils/effective-ram.js'; +import { granuleRatio } from './lbug/lbug-config.js'; import { resolveAnalysisResourceBudget, UNBOUNDED_GRAPH_POOL_MESSAGE, @@ -45,9 +46,7 @@ export function resolveHostAnalysisResources( const request: AnalysisResourceRequest = { effectiveMemoryBytes, availableCpus: Math.max(1, Math.floor(os.availableParallelism() / concurrency)), - totalMb: - options.totalMb ?? - (concurrency > 1 ? Math.floor((effectiveMemoryBytes / MIB) * 0.75) : undefined), + totalMb: options.totalMb, mainHeapMb: options.mainHeapMb, allowHeapOvercommit: options.mainHeapMb !== undefined && options.totalMb === undefined, workers: options.workers ?? positiveEnvironment('GITNEXUS_WORKER_POOL_SIZE'), @@ -58,6 +57,7 @@ export function resolveHostAnalysisResources( preferredWorkers: 16, includeParsing: options.includeParsing, graphPoolBytes: positiveEnvironment('GITNEXUS_LBUG_BUFFER_POOL_SIZE'), + graphPoolGranuleRatio: granuleRatio(), modelMemoryMb: options.includeEmbeddings && !http ? 2048 : 0, supervisorBytes: options.supervisorBytes, }; diff --git a/gitnexus/src/core/analysis-phase-supervisor.ts b/gitnexus/src/core/analysis-phase-supervisor.ts index 14f6aafbf..c88f1886e 100644 --- a/gitnexus/src/core/analysis-phase-supervisor.ts +++ b/gitnexus/src/core/analysis-phase-supervisor.ts @@ -4,6 +4,7 @@ import type { RepoMeta } from '../storage/repo-meta.js'; import { requireExclusiveIndexLock } from '../storage/index-lock.js'; import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { embeddingJobDirectory } from '../storage/embedding-job.js'; +import { loadMeta } from '../storage/repo-meta.js'; import { publishPreparedAnalysis, readPreparedAnalysis, @@ -107,21 +108,35 @@ export async function publishAnalysisAfterProcessExit( actions.signal?.throwIfAborted(); requireSameTarget(expected, await actions.resolveTarget()); const prepared = await readPreparedAnalysis(root); - if (prepared?.id !== expected.publicationId || prepared.writerPid !== writerPid) + if (prepared && (prepared.id !== expected.publicationId || prepared.writerPid !== writerPid)) + throw new Error( + 'Prepared graph attempt changed during handoff; artifacts retained; retry analysis', + ); + const metadata = prepared + ? await publishPreparedAnalysis(root, { + saveMetadata: actions.saveMetadata, + verify: async (file, metadata) => { + if (metadata.repoPath !== expected.repoPath || metadata.lastCommit !== expected.commit) + throw new Error('Prepared graph belongs to a different checkout'); + actions.signal?.throwIfAborted(); + await actions.verify(file, metadata); + actions.signal?.throwIfAborted(); + }, + }) + : await loadMeta(root); + // Another waiting writer may recover this exact receipt after the graph + // child exits. Adopt only the clean, already-published generation. + if ( + !metadata || + metadata.graphGeneration?.id !== expected.publicationId || + metadata.repoPath !== expected.repoPath || + metadata.lastCommit !== expected.commit || + metadata.incrementalInProgress || + metadata.graphWriteCollapsed + ) throw new Error( 'Prepared graph attempt changed during handoff; artifacts retained; retry analysis', ); - const metadata = await publishPreparedAnalysis(root, { - saveMetadata: actions.saveMetadata, - verify: async (file, metadata) => { - if (metadata.repoPath !== expected.repoPath || metadata.lastCommit !== expected.commit) - throw new Error('Prepared graph belongs to a different checkout'); - actions.signal?.throwIfAborted(); - await actions.verify(file, metadata); - actions.signal?.throwIfAborted(); - }, - }); - if (!metadata) throw new Error('Prepared graph disappeared during publication'); // Once publication starts, complete discoverability even if cancellation // arrived during its filesystem operations. Paid work remains inadmissible. const finalized = await actions.finalizeGraph(metadata); diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts index 71acc3342..657c6ed46 100644 --- a/gitnexus/src/core/analysis-resource-budget.ts +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -25,6 +25,8 @@ export interface AnalysisResourceRequest { /** Resume/sync can admit embeddings without reserving parser workers. */ includeParsing?: boolean; graphPoolBytes?: number; + /** LadybugDB COPY accounting uses OS-page-sized granules. */ + graphPoolGranuleRatio?: number; /** Resident parent process, charged once per child phase. */ supervisorBytes?: number; /** Estimated peak model/sidecar allocation; zero for an HTTP provider. */ @@ -101,9 +103,15 @@ export function resolveAnalysisResourceBudget( const supervisorMb = Math.ceil( integer(request.supervisorBytes ?? 0, 'supervisor reservation', 0) / MIB, ); + const poolScale = integer(request.graphPoolGranuleRatio ?? 1, 'graph pool granule ratio'); const graphPoolBytes = request.graphPoolBytes === undefined - ? bytes(Math.max(GRAPH_POOL_MB, Math.min(4096, Math.floor(totalMb * 0.15))), 'graph pool') + ? bytes( + automatic + ? Math.min(2048 * poolScale, Math.floor(availableMb * 0.8)) + : Math.max(GRAPH_POOL_MB * poolScale, Math.min(4096, Math.floor(totalMb * 0.15))), + 'graph pool', + ) : integer(request.graphPoolBytes, 'graph pool bytes', 8 * MIB); // Zero is the native engine's unbounded override, never a bounded allocation. const graphPoolMb = Math.ceil(graphPoolBytes / MIB); diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index e2b5a0a91..57ec1d2ec 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -119,6 +119,7 @@ export function createAutoSyncAnalysisRunner( const resources = options.resources ?? resolveHostAnalysisResources({ + mainHeapMb: resolveWorkerHeapMb(concurrency) + 48, includeEmbeddings: await analysisMayEmbed(repoPath, options), supervisorBytes: Math.ceil(analysisSupervisorReservation() / (concurrency ?? 1)), concurrency, diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index 81c66b68f..ca2eb871e 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -7,6 +7,8 @@ import { createEmbeddingJobDirectories, embeddingJobDirectory, syncEmbeddingJobDirectory, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, } from '../../storage/embedding-job.js'; import { adoptEmbeddingArtifact, @@ -30,6 +32,7 @@ import { LegacyVectorSource, legacyVectorCompatibility } from '../lbug/legacy-ve import { readLegacyEmbeddingSource } from '../../storage/legacy-embedding-source.js'; import { drainAnalysisNativeResources } from '../lbug/lbug-adapter.js'; import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; +import { estimateBufferPool } from '../lbug/lbug-config.js'; import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; import { embeddingJobIdentity } from './embedding-job-identity.js'; import { resolveEmbeddingIdentity, type EmbeddingIdentity } from './embedding-identity.js'; @@ -349,10 +352,15 @@ export async function runPublishedEmbeddingJob( if (!manifestExists && artifact?.owner.jobIdentity === identities.identity) throw new Error('Owned embedding manifest is missing; paid artifacts retained'); if (!manifestExists) await clearUnstartedManifest(directory); + const graphElements = (metadata.stats?.nodes ?? NaN) + (metadata.stats?.edges ?? NaN); + const sourcePoolBytes = + Number.isSafeInteger(graphElements) && graphElements >= 0 + ? Math.min(resources.graphPoolBytes, estimateBufferPool(graphElements)) + : resources.graphPoolBytes; const getSource = async (): Promise => (source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)( graphPath, - resources.graphPoolBytes, + sourcePoolBytes, resources.nativeThreads, )); const manifest = manifestExists @@ -429,6 +437,30 @@ export async function runPublishedEmbeddingJob( } signal?.throwIfAborted(); const artifactDirectory = await jobDirectory(root, owner.artifactIdentity); + if (owner.artifactIdentity !== owner.jobIdentity) { + // Rebased artifacts live apart from the journal. Keep independent proof + // of ownership so a later generation can reclaim their native payload. + try { + const receipt = (await readEmbeddingJobRecord(artifactDirectory, 'artifact.json')) as { + version: number; + identity: string; + jobIdentity: string; + }; + if ( + receipt.version !== 1 || + receipt.identity !== owner.artifactIdentity || + !/^[a-f0-9]{64}$/.test(receipt.jobIdentity) + ) + throw new Error('Embedding artifact has unknown ownership'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + await writeEmbeddingJobRecord(artifactDirectory, 'artifact.json', { + version: 1, + identity: owner.artifactIdentity, + jobIdentity: owner.jobIdentity, + }); + } + } const hasStore = await existsOwnedFile(path.join(artifactDirectory, 'vectors')); const hasJournal = await existsOwnedFile(path.join(directory, 'job.json')); if (!hasStore && hasJournal) @@ -560,6 +592,7 @@ export async function runPublishedEmbeddingJob( }, storeResources, options.metadata, + legacy?.identity, ); if (!published) return { metadata, alreadyComplete: false, phase: 'indexing', ...result }; if (result.state.excludedNodes) diff --git a/gitnexus/src/core/embeddings/vector-publication.ts b/gitnexus/src/core/embeddings/vector-publication.ts index 786a9859b..709979b0d 100644 --- a/gitnexus/src/core/embeddings/vector-publication.ts +++ b/gitnexus/src/core/embeddings/vector-publication.ts @@ -9,6 +9,7 @@ import { VectorStore } from '../lbug/vector-store.js'; import { syncPublishedMetadata } from '../analysis-publication.js'; import { getExactScanLimit } from '../platform/capabilities.js'; import { reclaimAnalysisArtifacts } from '../../storage/analysis-artifact-cleanup.js'; +import { readEmbeddingArtifactOwners } from '../../storage/embedding-artifact.js'; export interface VectorPublicationActions { loadMetadata: () => Promise; @@ -24,6 +25,7 @@ export async function publishVectorGeneration( generation: PublishedVectorGeneration, resources: { bufferPoolBytes: number; threads: number; batchBytes: number }, actions: VectorPublicationActions, + migratedLegacySource?: string, ): Promise { const exactScanLimit = getExactScanLimit(); if (generation.indexKind === 'exact' && generation.chunks > exactScanLimit) return null; @@ -89,6 +91,16 @@ export async function publishVectorGeneration( } finally { await store.close(); } + if (previous.legacyVectorSource && previous.legacyVectorSource === migratedLegacySource) { + const owners = await readEmbeddingArtifactOwners(metadataDirectory); + if ( + owners.some( + (owner) => owner.artifactIdentity === artifactIdentity && owner.phase === 'sealed', + ) && + !owners.some((owner) => owner.fallbackLegacy === previous.legacyVectorSource) + ) + delete metadata.legacyVectorSource; + } await actions.saveMetadata(metadata); await syncPublishedMetadata(metadataDirectory, metadata); await reclaimAnalysisArtifacts(metadataDirectory, metadata); diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index ac65721d5..24212b9d7 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -2065,13 +2065,15 @@ async function runFullAnalysisInner( const ftsDisabledReason = resolveFtsDisableReason(options.skipFts); const initAnalysisLbug = (dbPath: string) => { if (options.resources) { - // Admission is a ceiling. Preserve the existing graph-sized cache so - // small repositories do not pay for a host-sized native allocation. + // Only an explicit family budget caps the existing page-scaled sizing. + // Automatic reservations preserve the COPY floor on large-page hosts. const sizedPool = getEffectiveBufferPoolSize(); const ceiling = options.resources.parse.graphPoolBytes; return initLbug(dbPath, { skipFts: !!ftsDisabledReason, - bufferPoolBytes: Math.min(sizedPool || ceiling, ceiling), + bufferPoolBytes: options.resources.automatic + ? sizedPool + : Math.min(sizedPool || ceiling, ceiling), threads: options.resources.parse.nativeThreads, }); } diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index c55869f83..667254258 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -4592,11 +4592,17 @@ export class LocalBackend { let embeddedDims: number | undefined; try { if (repo.generationUnknown) { + if (degraded) + degraded.reason = + 'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.'; this.lastQueryEmbeddingDims.delete(repo.lbugPath); return []; } const metadata = await this.readToolMetadata(repo); if (repo.generationUnknown) { + if (degraded) + degraded.reason = + 'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.'; this.lastQueryEmbeddingDims.delete(repo.lbugPath); return []; } @@ -4610,6 +4616,8 @@ export class LocalBackend { if (degraded && selection.reason === 'embedding-pending') degraded.reason = 'Semantic search is unavailable while embeddings are pending. Run gitnexus analyze to resume; graph and keyword search remain available.'; + else if (degraded && selection.reason !== 'embeddings-unavailable') + degraded.reason = `Semantic search is unavailable (${selection.reason}). Run gitnexus analyze; graph and keyword search remain available.`; this.lastQueryEmbeddingDims.delete(repo.lbugPath); return []; } diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts index a79518976..a33cf7ff2 100644 --- a/gitnexus/src/server/analyze-launch.ts +++ b/gitnexus/src/server/analyze-launch.ts @@ -36,6 +36,7 @@ import type { RouteHookConfig } from '../core/ingestion/language-provider.js'; import { isTerminalJobStatus, type JobManager } from './analyze-job.js'; import type { WorkerMessage } from './analyze-worker.js'; import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; +import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; const _require = createRequire(import.meta.url); @@ -231,7 +232,10 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { const jobStartMs = Date.now(); const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); const resources = resolveHostAnalysisResources({ - mainHeapMb: Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb + 48 : undefined, + mainHeapMb: + (Number.isInteger(envHeapMb) && envHeapMb > 0 + ? envHeapMb + : Math.min(8192, autoHeapCapMb())) + 48, includeEmbeddings: await analysisMayEmbed(targetPath, opts), supervisorBytes: analysisSupervisorReservation(), }); diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index bd2639880..bf5864ea6 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -2243,7 +2243,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }); } catch (err: any) { const current = embedJobManager.getJob(job.id); - logger.error({ err, repo: entry.name }, 'Embedding generation failed'); + // Provider causes may contain response text. Log only the public + // message, as the CLI does, without pino's recursive Error serializer. + logger.error( + { reason: err.message || 'Embedding generation failed', repo: entry.name }, + 'Embedding generation failed', + ); if (!current || current.status !== 'failed') { embedJobManager.updateJob(job.id, { status: 'failed', diff --git a/gitnexus/src/storage/analysis-artifact-cleanup.ts b/gitnexus/src/storage/analysis-artifact-cleanup.ts index b32202316..5681f257a 100644 --- a/gitnexus/src/storage/analysis-artifact-cleanup.ts +++ b/gitnexus/src/storage/analysis-artifact-cleanup.ts @@ -46,7 +46,7 @@ export async function reclaimAnalysisArtifacts( keepJob(owner.artifactIdentity); keepJob(owner.jobIdentity); keepJob(owner.fallbackArtifact); - keepGraph(`lbug.generation.${owner.graphGeneration}`); + if (owner.phase === 'pending') keepGraph(`lbug.generation.${owner.graphGeneration}`); if (owner.fallbackLegacy) legacy.add(owner.fallbackLegacy); } // Validate every reference before the first deletion, including retired @@ -63,22 +63,26 @@ export async function reclaimAnalysisArtifacts( } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } - const orphanJobs: string[] = []; + const orphanJobs: Array<{ directory: string; receiptName: string }> = []; for (const name of jobEntries) { if (typeof name !== 'string' || !ID.test(name) || jobs.has(name)) continue; const job = await embeddingJobDirectory(path.join(embeddingRoot, name)); - try { - const receipt = (await readEmbeddingJobRecord(job, 'job.json')) as { - version?: number; - identity?: string; - }; - if (receipt.version !== 1 || receipt.identity !== name) - throw new Error('Unreferenced embedding job has unknown ownership'); - orphanJobs.push(job); - } catch (error) { - // An interrupted bootstrap has no paid work, but without a receipt we - // leave it to explicit cleanup instead of guessing ownership. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + for (const receiptName of ['job.json', 'artifact.json', 'retained.json']) { + try { + const receipt = (await readEmbeddingJobRecord(job, receiptName)) as { + version?: number; + identity?: string; + }; + // Older retained copies have no identity in their receipt. Keep + // those unverifiable directories, as with an interrupted bootstrap. + if (receiptName === 'retained.json' && receipt.identity === undefined) break; + if (receipt.version !== 1 || receipt.identity !== name) + throw new Error('Unreferenced embedding artifact has unknown ownership'); + orphanJobs.push({ directory: job, receiptName }); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } } } const remove = async (file: string, recursive = false): Promise => { @@ -103,19 +107,19 @@ export async function reclaimAnalysisArtifacts( if (await remove(path.join(root, graph))) for (const suffix of ['.wal', '.shadow']) await remove(path.join(root, graph + suffix)); } - for (const job of orphanJobs) { + for (const { directory: job, receiptName } of orphanJobs) { // Keep the ownership receipt until every payload is gone. Recursive // removal of the whole directory can delete job.json before a busy // native file fails on Windows, making the next cleanup unverifiable. let removed = true; for (const name of await fs.readdir(job)) { - if (name === 'job.json') continue; + if (name === receiptName) continue; if (!(await remove(path.join(job, name), true))) { removed = false; break; } } - if (removed && (await remove(path.join(job, 'job.json')))) await fs.rmdir(job); + if (removed && (await remove(path.join(job, receiptName)))) await fs.rmdir(job); } } catch (error) { warn( diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts index 846662630..13063c4b5 100644 --- a/gitnexus/src/storage/embedding-job.ts +++ b/gitnexus/src/storage/embedding-job.ts @@ -12,6 +12,7 @@ const RECORD_NAMES = new Set([ 'rebase.json', 'legacy.json', 'retained.json', + 'artifact.json', ]); const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; diff --git a/gitnexus/src/storage/shared-vector-artifact.ts b/gitnexus/src/storage/shared-vector-artifact.ts index dfc4348c4..1ee2aadc7 100644 --- a/gitnexus/src/storage/shared-vector-artifact.ts +++ b/gitnexus/src/storage/shared-vector-artifact.ts @@ -20,6 +20,7 @@ import { type FileIdentity = { dev: string; ino: string; size: string; mtimeNs: string }; interface RetentionReceipt { version: 1; + identity?: string; signature: string; source: FileIdentity; /** Absent until the entire retained file is durable. */ @@ -138,7 +139,7 @@ export async function retainSharedVectorArtifact( let intent: RetentionReceipt; try { await fs.mkdir(preparing, { mode: 0o700 }); - intent = { version: 1, signature, source: sourceIdentity }; + intent = { version: 1, identity: artifact, signature, source: sourceIdentity }; await writeEmbeddingJobRecord(preparing, 'retained.json', intent); await syncEmbeddingJobDirectory(parent); } catch (error) { @@ -148,7 +149,7 @@ export async function retainSharedVectorArtifact( } catch (readError) { if ((readError as NodeJS.ErrnoException).code !== 'ENOENT') throw readError; await recoverEmptyPreparation(preparing); - intent = { version: 1, signature, source: sourceIdentity }; + intent = { version: 1, identity: artifact, signature, source: sourceIdentity }; await writeEmbeddingJobRecord(preparing, 'retained.json', intent); await syncEmbeddingJobDirectory(parent); } diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts index 9a817172a..e84efee6c 100644 --- a/gitnexus/test/integration/embedding-job-controller.test.ts +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -486,6 +486,23 @@ describe('pending artifacts across graph generations', () => { }); describe('published graph embedding job controller', () => { + it('reclaims the prior rebased artifact after a third generation seals', async () => { + const f = await fixture(3); + await f.run(); + await f.nextGeneration(); + const second = await f.run(); + const previous = second.metadata.vectorGeneration!; + expect(previous.artifactIdentity).not.toBe(previous.identity); + const oldDirectory = path.join(f.root, 'embeddings', previous.artifactIdentity!); + expect( + JSON.parse(await fs.readFile(path.join(oldDirectory, 'artifact.json'), 'utf8')).payload + .identity, + ).toBe(previous.artifactIdentity); + await f.nextGeneration(); + const third = await f.run(); + expect(third.phase).toBe('ready'); + await expect(fs.stat(oldDirectory)).rejects.toMatchObject({ code: 'ENOENT' }); + }); it('resumes an interrupted bootstrap without rescanning the manifest or losing the pending job', async () => { const f = await fixture(3); const original = VectorStore.open.bind(VectorStore); diff --git a/gitnexus/test/integration/legacy-embedding-controller.test.ts b/gitnexus/test/integration/legacy-embedding-controller.test.ts index 88559ac55..31fe43a0d 100644 --- a/gitnexus/test/integration/legacy-embedding-controller.test.ts +++ b/gitnexus/test/integration/legacy-embedding-controller.test.ts @@ -272,6 +272,9 @@ describe('legacy migration through the published job controller', () => { const f = await fixture(5, ['Const:a.ts:n00002']); f.changed.set(f.id(1), 'const changed = 2;'); const result = await f.run(); + expect(result.metadata.legacyVectorSource).toBeUndefined(); + expect((await loadMeta(f.root))?.legacyVectorSource).toBeUndefined(); + await expect(fs.stat(f.sourcePath)).rejects.toMatchObject({ code: 'ENOENT' }); expect(result.state?.reusedNodes).toBe(3); expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(2); expect(f.events[0]).toContain('chunk size/overlap'); diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts index 5af29748b..a6dbf2bd7 100644 --- a/gitnexus/test/unit/analysis-host-resources.test.ts +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -3,6 +3,7 @@ import { analysisMayEmbed, resolveHostAnalysisResources, } from '../../src/core/analysis-host-resources.js'; +import { heapCapMbFor } from '../../src/core/ingestion/utils/effective-ram.js'; const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), placement: vi.fn() })); vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (original) => ({ @@ -47,8 +48,8 @@ it('reserves local model capacity only for invocations that may embed', () => { it('divides the host allowance across concurrent analyses', () => { const one = resolveHostAnalysisResources({ includeEmbeddings: false }); const four = resolveHostAnalysisResources({ includeEmbeddings: false, concurrency: 4 }); - expect(four.totalBytes * 4).toBe(24576 * 1024 ** 2); - expect(four.totalBytes * 4).toBeLessThanOrEqual(one.totalBytes); + expect(four.automatic).toBe(true); + expect(four.mainHeapMb).toBe(heapCapMbFor(8192 * 1024 ** 2)); expect(four.parse.nativeThreads * 4).toBe(one.parse.nativeThreads); expect(four.parse.workers).toBeLessThanOrEqual(3); }); diff --git a/gitnexus/test/unit/analysis-phase-supervisor.test.ts b/gitnexus/test/unit/analysis-phase-supervisor.test.ts index 600410468..321e29749 100644 --- a/gitnexus/test/unit/analysis-phase-supervisor.test.ts +++ b/gitnexus/test/unit/analysis-phase-supervisor.test.ts @@ -11,7 +11,10 @@ import { type PreparedAnalysisHandoff, } from '../../src/core/analysis-phase-supervisor.js'; import { waitForAnalysisProcessExit } from '../helpers/analysis-process-exit.js'; -import { readPreparedAnalysis } from '../../src/core/analysis-publication.js'; +import { + readPreparedAnalysis, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; import { acquireIndexLock } from '../../src/storage/index-lock.js'; import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import type { RepoMeta } from '../../src/storage/repo-meta.js'; @@ -101,6 +104,38 @@ async function fixture() { } describe('graph process handoff', () => { + it('continues when a waiting writer already recovered the exact publication', async () => { + const f = await fixture(); + f.release(); + await f.exited; + const recovered = await publishPreparedAnalysis(f.root, f.actions); + expect(await readPreparedAnalysis(f.root)).toBeUndefined(); + expect((await f.finish()).graphGeneration).toEqual(recovered?.graphGeneration); + expect(f.actions.finalizeGraph).toHaveBeenCalledOnce(); + expect(f.actions.verify).toHaveBeenCalledOnce(); + }); + + it.each(['foreign', 'dirty', 'collapsed', 'commit'] as const)( + 'rejects a %s publication without a receipt', + async (state) => { + const f = await fixture(); + f.release(); + await f.exited; + const recovered = await publishPreparedAnalysis(f.root, f.actions); + if (!recovered) throw new Error('Missing publication'); + await saveMeta(f.root, { + ...recovered, + ...(state === 'foreign' + ? { graphGeneration: { ...recovered.graphGeneration!, id: 'e'.repeat(32) } } + : {}), + ...(state === 'dirty' ? { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } } : {}), + ...(state === 'collapsed' ? { graphWriteCollapsed: { expected: 10, persisted: 1 } } : {}), + ...(state === 'commit' ? { lastCommit: 'f'.repeat(40) } : {}), + } as RepoMeta); + await expect(f.finish()).rejects.toThrow('attempt changed'); + expect(f.actions.finalizeGraph).not.toHaveBeenCalled(); + }, + ); it('keeps prepared nonterminal and publishes only after actual exit', async () => { const f = await fixture(); const provider = vi.fn(); diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts index 273882571..1f085565a 100644 --- a/gitnexus/test/unit/analysis-publication.test.ts +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -99,72 +99,83 @@ describe('durable graph publication', () => { expect(graphs).toContain((await loadMeta(root))?.graphPath); }); - it('reclaims unreferenced jobs while retaining all model owners and their fallback vectors', async () => { - const { root } = await fixture(); - const current = 'a'.repeat(64), - fallback = 'b'.repeat(64), - orphan = 'c'.repeat(64); - for (const id of [current, fallback, orphan]) { - const directory = path.join(root, 'embeddings', id); - await fs.mkdir(directory, { recursive: true }); - await writeEmbeddingJobRecord(directory, 'job.json', { version: 1, identity: id }); - await fs.writeFile(path.join(directory, 'vectors'), 'paid vectors'); - } - await adoptEmbeddingArtifact( - root, - { - vectorIdentity: 'd'.repeat(64), - inputIdentity: 'e'.repeat(64), - dimensions: 4, - schema: 'fixture', - }, - null, - { - artifactIdentity: current, - jobIdentity: current, - fallbackArtifact: fallback, - graphGeneration: 'f'.repeat(32), - sourceIdentity: '1'.repeat(64), - manifestDigest: '2'.repeat(64), - phase: 'pending', - revalidateInputs: false, - }, - async () => {}, - ); - await reclaimAnalysisArtifacts(root, metadata); - expect(await fs.readFile(path.join(root, 'embeddings', current, 'vectors'), 'utf8')).toBe( - 'paid vectors', - ); - expect(await fs.readFile(path.join(root, 'embeddings', fallback, 'vectors'), 'utf8')).toBe( - 'paid vectors', - ); - await expect(fs.stat(path.join(root, 'embeddings', orphan))).rejects.toMatchObject({ - code: 'ENOENT', - }); - }); + it.each(['pending', 'sealed'] as const)( + 'reclaims unreferenced jobs and obsolete graphs for a %s model owner', + async (phase) => { + const { root } = await fixture(); + const current = 'a'.repeat(64), + fallback = 'b'.repeat(64), + orphan = 'c'.repeat(64); + const ownerGraph = path.join(root, `lbug.generation.${'f'.repeat(32)}`); + await fs.writeFile(ownerGraph, 'old owner graph'); + for (const id of [current, fallback, orphan]) { + const directory = path.join(root, 'embeddings', id); + await fs.mkdir(directory, { recursive: true }); + await writeEmbeddingJobRecord(directory, 'job.json', { version: 1, identity: id }); + await fs.writeFile(path.join(directory, 'vectors'), 'paid vectors'); + } + await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: current, + jobIdentity: current, + fallbackArtifact: fallback, + graphGeneration: 'f'.repeat(32), + sourceIdentity: '1'.repeat(64), + manifestDigest: '2'.repeat(64), + phase, + revalidateInputs: false, + }, + async () => {}, + ); + await reclaimAnalysisArtifacts(root, metadata); + if (phase === 'pending') + expect(await fs.readFile(ownerGraph, 'utf8')).toBe('old owner graph'); + else await expect(fs.stat(ownerGraph)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(path.join(root, 'embeddings', current, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + expect(await fs.readFile(path.join(root, 'embeddings', fallback, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + await expect(fs.stat(path.join(root, 'embeddings', orphan))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }, + ); - it('retains an obsolete job receipt until its busy vector file can be removed', async () => { - const { root } = await fixture(); - const identity = 'e'.repeat(64); - const job = path.join(root, 'embeddings', identity); - await fs.mkdir(job, { recursive: true }); - await writeEmbeddingJobRecord(job, 'job.json', { version: 1, identity }); - const vectors = path.join(job, 'vectors'); - await fs.writeFile(vectors, 'obsolete vectors'); - const remove = fs.rm.bind(fs); - const busy = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { - if (String(file) === vectors) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); - return remove(file, options); - }); - const warn = vi.fn(); - await reclaimAnalysisArtifacts(root, metadata, undefined, warn); - expect(await fs.readFile(path.join(job, 'job.json'), 'utf8')).toContain(identity); - expect(await fs.readFile(vectors, 'utf8')).toBe('obsolete vectors'); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('cleanup will retry')); - busy.mockRestore(); - await reclaimAnalysisArtifacts(root, metadata); - await expect(fs.stat(job)).rejects.toMatchObject({ code: 'ENOENT' }); - }); + it.each(['job.json', 'artifact.json', 'retained.json'])( + 'retains %s until its busy vector file can be removed', + async (receiptName) => { + const { root } = await fixture(); + const identity = 'e'.repeat(64); + const job = path.join(root, 'embeddings', identity); + await fs.mkdir(job, { recursive: true }); + await writeEmbeddingJobRecord(job, receiptName, { version: 1, identity }); + const vectors = path.join(job, 'vectors'); + await fs.writeFile(vectors, 'obsolete vectors'); + const remove = fs.rm.bind(fs); + const busy = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === vectors) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + return remove(file, options); + }); + const warn = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, warn); + expect(await fs.readFile(path.join(job, receiptName), 'utf8')).toContain(identity); + expect(await fs.readFile(vectors, 'utf8')).toBe('obsolete vectors'); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('cleanup will retry')); + busy.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata); + await expect(fs.stat(job)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); it('keeps the only legacy vector source while a model-space job references it', async () => { const { root } = await fixture(); diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts index 524141cc2..9f3fa0e78 100644 --- a/gitnexus/test/unit/analysis-resource-budget.test.ts +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -9,6 +9,27 @@ const MIB = 1024 * 1024; const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; describe('analysis resource admission', () => { + it.each([1, 4, 16])( + 'preserves the automatic graph cap and explicit COPY floor at granule ratio %s', + (scale) => { + const automatic = resolveAnalysisResourceBudget({ ...request, graphPoolGranuleRatio: scale }); + expect(automatic.parse.graphPoolBytes).toBe( + Math.min(2048 * scale, Math.floor(32768 * 0.8)) * MIB, + ); + const explicit = resolveAnalysisResourceBudget({ + ...request, + totalMb: 16384, + graphPoolGranuleRatio: scale, + }); + expect(explicit.parse.graphPoolBytes).toBeGreaterThanOrEqual(256 * scale * MIB); + expect(explicit.parse.reservedBytes).toBeLessThanOrEqual(explicit.totalBytes); + }, + ); + it('rejects an explicit family budget that cannot fit the 64 KiB COPY floor', () => { + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 2048, graphPoolGranuleRatio: 16 }), + ).toThrow('cannot admit'); + }); it.each([4096, 8192, 32768])( 'preserves automatic main-heap and worker capacity on a %s MiB host', (memoryMb) => { diff --git a/gitnexus/test/unit/analyze-launch-collapse.test.ts b/gitnexus/test/unit/analyze-launch-collapse.test.ts index 86554da0f..1b9a32f89 100644 --- a/gitnexus/test/unit/analyze-launch-collapse.test.ts +++ b/gitnexus/test/unit/analyze-launch-collapse.test.ts @@ -19,6 +19,9 @@ */ import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; import { EventEmitter } from 'node:events'; +import * as hostResources from '../../src/core/analysis-host-resources.js'; +import * as effectiveRam from '../../src/core/ingestion/utils/effective-ram.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; // `vi.mock` factories are hoisted above every top-level `const`, and this file // imports the module under test statically — so anything a factory closes over @@ -209,6 +212,39 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () ); }); + it.each([ + [undefined, 8192], + ['4096', 4096], + ] as const)('preserves the server heap cap with override %s', async (override, heap) => { + vi.stubEnv('GITNEXUS_SERVER_ANALYZE_HEAP_MB', override); + const cap = vi.spyOn(effectiveRam, 'autoHeapCapMb').mockReturnValue(24576); + const admission = vi + .spyOn(hostResources, 'resolveHostAnalysisResources') + .mockImplementation((options) => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 32 * 1024 ** 3, + availableCpus: 8, + mainHeapMb: options.mainHeapMb, + modelMemoryMb: 0, + }), + ); + try { + expect((await runWorker(completeMessage()))?.status).toBe('complete'); + expect(admission).toHaveBeenCalledWith(expect.objectContaining({ mainHeapMb: heap + 48 })); + expect(forkMock).toHaveBeenCalledWith( + expect.any(String), + [], + expect.objectContaining({ + execArgv: expect.arrayContaining([`--max-old-space-size=${heap}`]), + }), + ); + } finally { + cap.mockRestore(); + admission.mockRestore(); + vi.unstubAllEnvs(); + } + }); + it('publishes the graph before acknowledging paid work and waits for both exits', async () => { child.pid = 12345; const writer = child; diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 4ce6da601..32645f982 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -3,6 +3,7 @@ import { EventEmitter } from 'node:events'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { _captureLogger } from '../../src/core/logger.js'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ @@ -232,27 +233,44 @@ describe('POST /api/embed staged recovery preflight', () => { contentRetention: 'full', stats: { embeddings: 3 }, }); - mocks.runAnalysisEmbeddingPhase.mockRejectedValueOnce(new Error('source fingerprint mismatch')); - await invoke('/api/embed'); - await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); - // The shared phase reads source identity from this slot's published metadata. - // embedding-job-controller.test.ts exercises that read with the real controller. - expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - metadataDirectory: entry.storagePath, - graphPath: path.join(entry.storagePath, 'lbug'), - nodeLimit: 0, - resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), - signal: expect.any(AbortSignal), - }), - ); - expect(mocks.runEmbeddingPipeline).not.toHaveBeenCalled(); - expect(entry.path).not.toBe(entry.storagePath); - expect(mocks.updateJob).toHaveBeenCalledWith( - 'embed-job', - expect.objectContaining({ status: 'failed', error: 'source fingerprint mismatch' }), - ); - expect(mocks.saveMeta).not.toHaveBeenCalled(); + const log = _captureLogger(); + try { + mocks.runAnalysisEmbeddingPhase.mockRejectedValueOnce( + new Error('source fingerprint mismatch', { + cause: new Error('private provider response excerpt'), + }), + ); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); + // The shared phase reads source identity from this slot's published metadata. + // embedding-job-controller.test.ts exercises that read with the real controller. + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + metadataDirectory: entry.storagePath, + graphPath: path.join(entry.storagePath, 'lbug'), + nodeLimit: 0, + resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), + signal: expect.any(AbortSignal), + }), + ); + expect(mocks.runEmbeddingPipeline).not.toHaveBeenCalled(); + expect(entry.path).not.toBe(entry.storagePath); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'failed', error: 'source fingerprint mismatch' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + expect(log.records()).toContainEqual( + expect.objectContaining({ + reason: 'source fingerprint mismatch', + repo: entry.name, + msg: 'Embedding generation failed', + }), + ); + expect(log.text()).not.toContain('private provider response excerpt'); + } finally { + log.restore(); + } }); afterEach(() => { diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index d57bc6ec2..589785090 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -7,6 +7,8 @@ import { type AutoSyncAnalysisRunner, } from '../../src/core/auto-sync/analysis-worker-launch.js'; import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import * as hostResources from '../../src/core/analysis-host-resources.js'; +import * as effectiveRam from '../../src/core/ingestion/utils/effective-ram.js'; function createAutoSyncAnalysisRunner(overrides: Parameters[0]) { const run = createRunner(overrides); @@ -45,6 +47,44 @@ function createChild() { } describe('auto-sync analysis worker', () => { + it.each([ + [1, 8192], + [4, 3072], + ])('admits the historical worker heap for %s concurrent repositories', async (slots, heap) => { + const cap = vi.spyOn(effectiveRam, 'autoHeapCapMb').mockReturnValue(12288); + const mayEmbed = vi.spyOn(hostResources, 'analysisMayEmbed').mockResolvedValue(false); + const admission = vi + .spyOn(hostResources, 'resolveHostAnalysisResources') + .mockImplementation((options) => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 32 * 1024 ** 3, + availableCpus: 8, + mainHeapMb: options.mainHeapMb, + modelMemoryMb: 0, + }), + ); + const child = createChild(); + const forkWorker = vi.fn(() => child as any); + const run = createRunner({ forkWorker }); + try { + const pending = run('/tmp/repo', {}, 30_000, undefined, undefined, slots); + await vi.waitFor(() => expect(forkWorker).toHaveBeenCalledOnce()); + expect(admission).toHaveBeenCalledWith( + expect.objectContaining({ mainHeapMb: heap + 48, concurrency: slots }), + ); + expect(forkWorker).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining([`--max-old-space-size=${heap}`]), + ); + child.emit('message', { type: 'complete', result: { stats: { files: 1 } } }); + child.emit('exit', 0, null); + await expect(pending).resolves.toEqual({ stats: { files: 1 } }); + } finally { + cap.mockRestore(); + mayEmbed.mockRestore(); + admission.mockRestore(); + } + }); it('continues after actual exit and retains the original timeout across phases', async () => { const graph = createChild(); const embedding = createChild(); diff --git a/gitnexus/test/unit/local-backend-vector-generation.test.ts b/gitnexus/test/unit/local-backend-vector-generation.test.ts index e57d95da9..135cbfc29 100644 --- a/gitnexus/test/unit/local-backend-vector-generation.test.ts +++ b/gitnexus/test/unit/local-backend-vector-generation.test.ts @@ -45,6 +45,46 @@ beforeEach(() => { }); describe('MCP generation gate', () => { + it.each([ + 'graph-generation-unverified', + 'vector-generation-mismatch', + 'vector-index-pending', + 'embeddings-empty', + 'embeddings-unavailable', + ])('reports semantic unavailability for %s except a never-embedded index', async (reason) => { + const backend = new LocalBackend() as unknown as Reader; + const meta: RepoMeta = { + ...metadata, + embeddingJob: undefined, + vectorGeneration: { + version: 1, + identity: 'a'.repeat(64), + graphGeneration: generation, + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', + }, + }; + if (reason === 'graph-generation-unverified') meta.graphGeneration = undefined; + if (reason === 'vector-generation-mismatch') + meta.vectorGeneration!.graphGeneration = 'd'.repeat(32); + if (reason === 'vector-index-pending') meta.vectorGeneration!.chunks = 1_000_000_000; + if (reason === 'embeddings-empty') + Object.assign(meta.vectorGeneration!, { nodes: 0, chunks: 0 }); + if (reason === 'embeddings-unavailable') meta.vectorGeneration = undefined; + vi.spyOn(backend, 'readToolMetadata').mockResolvedValue(meta); + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch({ lbugPath: graphPath }, 'hello', 5, degraded)).toEqual([]); + if (reason === 'embeddings-unavailable') expect(degraded.reason).toBeUndefined(); + else { + expect(degraded.reason).toContain(reason); + expect(degraded.reason).toContain('gitnexus analyze'); + } + expect(mocks.embed).not.toHaveBeenCalled(); + }); it('explains the temporary keyword-only lane while vectors are pending', async () => { const backend = new LocalBackend() as unknown as Reader; const degraded: { reason?: string } = {}; @@ -64,7 +104,10 @@ describe('MCP generation gate', () => { return metadata; }); } - expect(await backend.semanticSearch(repo, 'hello', 5)).toEqual([]); + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch(repo, 'hello', 5, degraded)).toEqual([]); + expect(degraded.reason).toContain('graph generation is unverified'); + expect(degraded.reason).toContain('gitnexus analyze'); expect(backend.lastQueryEmbeddingDims.has(graphPath)).toBe(false); expect(mocks.embed).not.toHaveBeenCalled(); expect(mocks.query).not.toHaveBeenCalled(); From 4a8325bc809df50bc1e2b4e68852c8e0efa5deac Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 16:53:46 +0000 Subject: [PATCH 49/67] perf(analyze): defer unused parser imports --- gitnexus/src/core/embeddings/ast-utils.ts | 4 ++- .../core/embeddings/structural-extractor.ts | 10 +++--- gitnexus/src/core/run-analyze.ts | 2 +- .../test/unit/embedding-inputs-lazy.test.ts | 32 +++++++++++++++++++ 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 gitnexus/test/unit/embedding-inputs-lazy.test.ts diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts index 9306a8fd6..2f4bac24e 100644 --- a/gitnexus/src/core/embeddings/ast-utils.ts +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -10,7 +10,6 @@ import { resolveLanguageKey, } from '../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; -import { getLanguageForFileContent, getProvider } from '../ingestion/languages/index.js'; import { extractNotebookPython, isNotebookPath } from '../ingestion/ipynb-extractor.js'; const parserCache = new Map(); @@ -20,6 +19,9 @@ const parserCache = new Map(); * Returns null if language is unavailable or parsing fails. */ export const ensureAndParse = async (content: string, filePath: string): Promise => { + // Fixed chunks need no provider registry. Load it only when parsing a snippet. + const { getLanguageForFileContent, getProvider } = + await import('../ingestion/languages/index.js'); // Same classifier as ingest. Filename-only maps `.h` → C++, so Objective-C // headers (and method snippets from those headers) would parse with the // wrong grammar and miss class_interface / protocol_declaration / methods. diff --git a/gitnexus/src/core/embeddings/structural-extractor.ts b/gitnexus/src/core/embeddings/structural-extractor.ts index a5799cfb4..85d949d9f 100644 --- a/gitnexus/src/core/embeddings/structural-extractor.ts +++ b/gitnexus/src/core/embeddings/structural-extractor.ts @@ -5,11 +5,9 @@ * to extract method and field names for embedding text generation. */ -import { getProviderForFileContent } from '../ingestion/languages/index.js'; import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js'; import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js'; import type { LanguageProvider } from '../ingestion/language-provider.js'; -import { buildTypeEnv } from '../ingestion/type-env.js'; import { SupportedLanguages } from 'gitnexus-shared'; import { ensureAndParse, findDeclarationNode } from './ast-utils.js'; @@ -34,6 +32,7 @@ export const extractStructuralNames = async ( parsedTree?: Awaited>, expectedName?: string, ): Promise => { + const { getProviderForFileContent } = await import('../ingestion/languages/index.js'); const provider = getProviderForFileContent(filePath, content); if (!provider) return { methodNames: [], fieldNames: [] }; @@ -47,7 +46,7 @@ export const extractStructuralNames = async ( const language = provider.id; const methodNames = extractMethodNames(classNode, provider, filePath, language); - const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language); + const fieldNames = await extractFieldNames(classNode, provider, tree, filePath, language); return { methodNames, fieldNames }; }; @@ -67,15 +66,16 @@ function extractMethodNames( return result.methods.map((m) => m.name); } -function extractFieldNames( +async function extractFieldNames( classNode: any, provider: LanguageProvider, tree: any, filePath: string, language: SupportedLanguages, -): string[] { +): Promise { if (!provider.fieldExtractor) return []; + const { buildTypeEnv } = await import('../ingestion/type-env.js'); const typeEnv = buildTypeEnv(tree, language); const context: FieldExtractorContext = { diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 24212b9d7..471f3dd33 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -256,7 +256,6 @@ import { type PersistedSpringConfigConsumerRow, } from './incremental/spring-config-drift.js'; import { shadowCandidatesFor } from './incremental/shadow-candidates.js'; -import { collectResolutionGroupFiles } from './incremental/resolution-groups.js'; import { shouldEscalateIncrementalWrite } from './incremental/escalation-gate.js'; import { ftsTablesAmong, @@ -4164,6 +4163,7 @@ async function runFullAnalysisInner( // Resolution groups can change edges between unchanged files without an // import or a new-graph edge touching the changed/deleted member. Expand // before the importer walk so dependents of these siblings refresh too. + const { collectResolutionGroupFiles } = await import('./incremental/resolution-groups.js'); for (const filePath of collectResolutionGroupFiles( [...hashDiff.toWrite, ...hashDiff.deleted], allFilePaths, diff --git a/gitnexus/test/unit/embedding-inputs-lazy.test.ts b/gitnexus/test/unit/embedding-inputs-lazy.test.ts new file mode 100644 index 000000000..00a6122f2 --- /dev/null +++ b/gitnexus/test/unit/embedding-inputs-lazy.test.ts @@ -0,0 +1,32 @@ +import { expect, it, vi } from 'vitest'; + +const registry = vi.hoisted(() => ({ loaded: false })); +vi.mock('../../src/core/ingestion/languages/index.js', () => { + registry.loaded = true; + throw new Error('The provider registry is unavailable in this fixture'); +}); + +import { iterateEmbeddingInputs } from '../../src/core/embeddings/embedding-inputs.js'; +import { DEFAULT_EMBEDDING_CONFIG } from '../../src/core/embeddings/types.js'; + +it('generates fixed function inputs without initializing language providers', async () => { + const content = 'export function value() { return 1; }'; + const inputs = []; + for await (const input of iterateEmbeddingInputs( + { + id: 'Function:value.ts:value', + label: 'Function', + name: 'value', + filePath: 'value.ts', + content, + startLine: 0, + endLine: 0, + }, + { ...DEFAULT_EMBEDDING_CONFIG, chunkStrategy: 'fixed' }, + )) { + inputs.push(input); + } + expect(inputs).toHaveLength(1); + expect(inputs[0].text).toContain(content); + expect(registry.loaded).toBe(false); +}); From ee07f66acf7e659530bb451ce83616659b1dab8a Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 16:59:36 +0000 Subject: [PATCH 50/67] fix(workers): retain failed shutdown reservations --- .../src/core/ingestion/workers/worker-pool.ts | 26 ++++++++++++++++++- gitnexus/src/storage/vector-generation.ts | 5 ++-- .../unit/worker-pool-timeout-retire.test.ts | 24 +++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index 1b521eed4..bd59b87ec 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -1195,7 +1195,31 @@ export const createWorkerPool = ( ): Promise => { const retired = Array.from(retiredWorkers); await Promise.all([ - ...liveWorkers.map((worker) => worker?.terminate().catch(() => undefined)), + ...liveWorkers.map(async (worker, workerIndex) => { + if (!worker) return; + let completeRelease!: () => void; + const termination = new Promise((resolve) => { + completeRelease = resolve; + }); + const released = () => { + worker.off('exit', released); + pendingWorkerReleases.delete(termination); + completeRelease(); + }; + if (requireDrain) { + pendingWorkerReleases.set(termination, workerIndex); + worker.once('exit', released); + } + try { + await worker.terminate(); + released(); + } catch { + // A rejected request does not prove physical exit. Retain ownership + // after workers[] is cleared, and release it on the eventual exit. + if (!requireDrain) released(); + else worker.unref?.(); + } + }), ...retired.map(async (record) => { // #2432: a retired worker that has not reached a JS-visible safe // point may be inside an N-API call — terminating it aborts the diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts index b66088a2f..81384eda5 100644 --- a/gitnexus/src/storage/vector-generation.ts +++ b/gitnexus/src/storage/vector-generation.ts @@ -125,8 +125,9 @@ export function selectVectorGeneration( }; } -/** Refuse ancestor symlinks as well as a symlinked database. The coordinator - * holds the slot lock while creating jobs; published directories are immutable. */ +/** Refuse symlinks from the owned metadata slot down to the database. Parent + * path aliases remain valid. The coordinator holds the slot lock while creating + * jobs; published directories are immutable. */ export async function assertOwnedVectorGeneration( metadataDirectory: string, identity: string, diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index 3ef56c54a..5ef639491 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -94,6 +94,30 @@ afterEach(() => { }); describe('worker pool timeout retirement', () => { + it('retains a live worker when its shutdown termination request rejects', async () => { + let live!: TimeoutThenHealthyWorker; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + workerFactory: () => { + live = new TimeoutThenHealthyWorker(); + live.terminate = async () => { + live.terminateCalls++; + throw new Error('termination rejected'); + }; + return live as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(live.terminateCalls).toBe(1); + live.emit('exit', 0); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + live.emit('exit', 0); + await pool.terminate(); + } + }); it('retains a rejected termination reservation until physical exit, then releases it', async () => { let broken: TimeoutThenHealthyWorker | undefined; const pool = createWorkerPool(workerUrl, 1, { From fdbf2d003f8baa05978b359187f4984a093020da Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 17:29:49 +0000 Subject: [PATCH 51/67] perf: skip unused budget supervisor receipt --- gitnexus/src/cli/index.ts | 12 +++++++++++- gitnexus/test/unit/analyze-bootstrap.test.ts | 12 +++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 0f0479c78..6168dc9ff 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -295,7 +295,17 @@ program // and embeds it. Its analyzer receipt is never used: the graph writer has // already finalized provenance. Keep the native check, without repeating // the complete analyzer identity snapshot in this separate process. - if (process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected) { + // An explicit heap budget without the budget-child marker always re-execs + // in ensureHeap before runFullAnalysis. Its parent receipt is unused too; + // the child still captures provenance before importing the analyzer. + const budgetSupervisor = + !options.watch && + options.memoryBudget !== undefined && + process.env.GITNEXUS_HEAP_LIMIT_SOURCE !== 'budget'; + if ( + budgetSupervisor || + (process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected) + ) { await createLbugLazyAction(() => import('./analyze.js'), 'analyzeCommand')( inputPath, options, diff --git a/gitnexus/test/unit/analyze-bootstrap.test.ts b/gitnexus/test/unit/analyze-bootstrap.test.ts index 80a816dc2..c24f32b17 100644 --- a/gitnexus/test/unit/analyze-bootstrap.test.ts +++ b/gitnexus/test/unit/analyze-bootstrap.test.ts @@ -46,13 +46,19 @@ afterEach(() => { }); it.each([ - { continuation: false, watch: false, captures: true }, - { continuation: false, watch: true, captures: true }, - { continuation: true, watch: false, captures: false }, + { continuation: false, watch: false, budget: false, source: '', captures: true }, + { continuation: false, watch: true, budget: false, source: '', captures: true }, + { continuation: true, watch: false, budget: false, source: '', captures: false }, + { continuation: false, watch: false, budget: true, source: '', captures: false }, + { continuation: false, watch: false, budget: true, source: 'auto', captures: false }, + { continuation: false, watch: false, budget: true, source: 'budget', captures: true }, + { continuation: false, watch: true, budget: true, source: '', captures: true }, ])('selects the analyzer bootstrap for %j', async (scenario) => { vi.stubEnv('GITNEXUS_ANALYSIS_CONTINUATION', scenario.continuation ? '1' : ''); + vi.stubEnv('GITNEXUS_HEAP_LIMIT_SOURCE', scenario.source); process.argv = [process.execPath, 'gitnexus', 'analyze', '/fixture']; if (scenario.watch) process.argv.push('--watch'); + if (scenario.budget) process.argv.push('--memory-budget', '2048'); await import('../../src/cli/index.js'); if (scenario.captures) { From 0c91345abf033ffdbca79b3dd3daea675cc59d04 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 17:39:37 +0000 Subject: [PATCH 52/67] fix: bound rejection history and watch budgets --- gitnexus/src/cli/analyze-watch.ts | 5 +- .../core/embeddings/embedding-job-runner.ts | 30 ++++++--- gitnexus/src/storage/embedding-job-journal.ts | 35 +++++++--- .../test/unit/embedding-job-journal.test.ts | 65 ++++++++++++++----- .../unit/watch-supervisor-reservation.test.ts | 29 +++++---- 5 files changed, 117 insertions(+), 47 deletions(-) diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index a281ce9ea..30352ac4a 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -199,7 +199,10 @@ export async function resolveWatchOptions( mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), workers: workerPoolSize, includeEmbeddings: await analysisMayEmbed(repoPath, { branch }), - supervisorBytes: Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES ?? 0), + supervisorBytes: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES), }); setEnvironment( diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index e705a075d..f4f8a3eef 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -2,10 +2,11 @@ import { createHash } from 'node:crypto'; import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; import type { VectorNodeReceipt, VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; -import type { - EmbeddingJobJournal, - EmbeddingJobState, - EmbeddingUnitIntent, +import { + embeddingInputFailureKey, + type EmbeddingJobJournal, + type EmbeddingJobState, + type EmbeddingUnitIntent, } from '../../storage/embedding-job-journal.js'; import { fetchEmbeddingWorkUnit, @@ -99,10 +100,23 @@ export async function runEmbeddingJob( intent: EmbeddingUnitIntent, ): Promise => { signal?.throwIfAborted(); - const history = journal.snapshot().active?.inputFailures ?? intent.inputFailures ?? {}; - if (Object.keys(history).some((id) => !unit.nodes.some((node) => node.id === id))) - throw new Error('Embedding rejection history does not match its manifest unit'); - const excluded = new Set(Object.keys(history).filter((id) => history[id] >= 3)); + const active = journal.snapshot().active ?? intent; + const history = active.inputFailures ?? {}; + const excluded = new Set(); + if (Object.keys(history).length) { + const nodesByKey = new Map( + unit.nodes.map((node) => [ + active.inputFailureKeyFormat === 'sha256' ? embeddingInputFailureKey(node.id) : node.id, + node.id, + ]), + ); + for (const [key, attempts] of Object.entries(history)) { + const id = nodesByKey.get(key); + if (id === undefined) + throw new Error('Embedding rejection history does not match its manifest unit'); + if (attempts >= 3) excluded.add(id); + } + } const complete = options.revalidateStoredInputs ? new Map() : await store.completedNodes(unit.nodes.map((node) => node.id)); diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts index b62b980cb..cb5d74408 100644 --- a/gitnexus/src/storage/embedding-job-journal.ts +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -1,7 +1,7 @@ /** Compact scheduling journal. The caller owns the checkout's exclusive job lock. */ import fs from 'node:fs/promises'; import path from 'node:path'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import type { EmbeddingManifestCursor, EmbeddingManifestReceipt, @@ -20,6 +20,8 @@ export interface EmbeddingUnitIntent { origin: 'manifest' | 'failed'; /** Bounded by the unit's 64 nodes; three deterministic rejections exclude a node. */ inputFailures?: Record; + /** Older receipts use raw node IDs. New receipts bound keys independently of ID length. */ + inputFailureKeyFormat?: 'sha256'; } export interface EmbeddingJobState { @@ -59,6 +61,9 @@ const integer = (value: unknown): value is number => const filename = (unit: number): string => `failed-${String(unit).padStart(16, '0')}.json`; const clone = (value: T): T => structuredClone(value); +export const embeddingInputFailureKey = (nodeId: string): string => + createHash('sha256').update(nodeId).digest('hex'); + /** Operational failures (permissions, I/O) must not be treated as lost state. */ function damagedRecord(error: unknown): boolean { const code = (error as NodeJS.ErrnoException).code; @@ -84,7 +89,8 @@ function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): !integer(intent.nodes) || intent.nodes < 1 || intent.nodes > 64 || - !['manifest', 'failed'].includes(intent.origin) + !['manifest', 'failed'].includes(intent.origin) || + (intent.inputFailureKeyFormat !== undefined && intent.inputFailureKeyFormat !== 'sha256') ) throw new Error('Invalid embedding unit intent'); if ( @@ -97,6 +103,7 @@ function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): ([id, attempts]) => !id.length || id.length > 1024 * 1024 || + (intent.inputFailureKeyFormat === 'sha256' && !DIGEST.test(id)) || !integer(attempts) || attempts < 1 || attempts > 3, @@ -331,6 +338,7 @@ export class EmbeddingJobJournal { next.failedUnits--; next.failedNodes -= requested.nodes; requested.inputFailures = record.intent.inputFailures; + requested.inputFailureKeyFormat = record.intent.inputFailureKeyFormat; } next.active = requested; }); @@ -388,21 +396,30 @@ export class EmbeddingJobJournal { }); } - /** Persist input-only failures before discarding rejected nodes' partial rows. */ + /** Persist input-only failures before discarding partial rows; return counts for these IDs. */ async recordInputRejections(ids: readonly string[]): Promise>> { - if (!ids.length) return clone(this.state.active?.inputFailures ?? {}); + if (!ids.length) return {}; await this.mutate(async (next) => { - if (!next.active || new Set(ids).size !== ids.length || ids.length > next.active.nodes) + const active = next.active; + if (!active || new Set(ids).size !== ids.length || ids.length > active.nodes) throw new Error('Invalid embedding input rejections'); - const history = { ...next.active.inputFailures }; + const history = Object.fromEntries( + Object.entries(active.inputFailures ?? {}).map(([key, attempts]) => [ + active.inputFailureKeyFormat === 'sha256' ? key : embeddingInputFailureKey(key), + attempts, + ]), + ); for (const id of ids) { if (!id.length || id.length > 1024 * 1024 || id === '__proto__') throw new Error('Invalid rejected node identity'); - history[id] = Math.min(3, (history[id] ?? 0) + 1); + const key = embeddingInputFailureKey(id); + history[key] = Math.min(3, (history[key] ?? 0) + 1); } - next.active.inputFailures = history; + active.inputFailures = history; + active.inputFailureKeyFormat = 'sha256'; }); - return clone(this.state.active!.inputFailures!); + const history = this.state.active!.inputFailures!; + return Object.fromEntries(ids.map((id) => [id, history[embeddingInputFailureKey(id)]])); } private async readFailed(name: string): Promise { diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts index 9f53ce013..fb38aea3d 100644 --- a/gitnexus/test/unit/embedding-job-journal.test.ts +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { EmbeddingJobJournal, + embeddingInputFailureKey, type EmbeddingUnitIntent, } from '../../src/storage/embedding-job-journal.js'; import { @@ -47,26 +48,56 @@ afterEach(async () => { }); describe('durable embedding scheduling', () => { - it('retains deterministic rejection history across a crash before vector exclusion', async () => { - let journal = await job(); - const id = 'Function:a.ts:bad'; - await journal.begin(unit(0)); - for (let attempt = 1; attempt <= 3; attempt++) { - await journal.recordInputRejections([id]); - journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); - expect(journal.snapshot().active?.inputFailures).toEqual({ [id]: attempt }); - if (attempt < 3) { - await journal.fail('provider'); + it.each(['Function:a.ts:bad', `Function:a.ts:${'x'.repeat(40_000)}`])( + 'retains rejection history across crashes for node ID %#', + async (id) => { + let journal = await job(); + await journal.begin(unit(0)); + for (let attempt = 1; attempt <= 3; attempt++) { + expect(await journal.recordInputRejections([id])).toEqual({ [id]: attempt }); journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); - await journal.begin(unit(0, 'failed')); + expect(journal.snapshot().active).toMatchObject({ + inputFailureKeyFormat: 'sha256', + inputFailures: { [embeddingInputFailureKey(id)]: attempt }, + }); + if (attempt < 3) { + await journal.fail('provider'); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + } } - } + await journal.acknowledge({ reusedNodes: 1, chunks: 1 }); + expect(journal.snapshot()).toMatchObject({ + excludedNodes: 1, + reusedNodes: 1, + completedNodes: 0, + }); + }, + ); + + it('upgrades existing raw-ID rejection history without resetting attempts', async () => { + let journal = await job(); + const id = 'Function:a.ts:legacy'; + await journal.begin({ ...unit(0), inputFailures: { [id]: 2 } }); + await journal.fail('input'); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + expect(await journal.recordInputRejections([id])).toEqual({ [id]: 3 }); await journal.acknowledge({ reusedNodes: 1, chunks: 1 }); - expect(journal.snapshot()).toMatchObject({ - excludedNodes: 1, - reusedNodes: 1, - completedNodes: 0, - }); + expect(journal.snapshot().excludedNodes).toBe(1); + }); + + it('fits a full unit of long rejected IDs within the receipt limit', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-journal-wide-')); + directories.push(directory); + const wideManifest = { ...manifest, totalNodes: 64, totalUnits: 1, bytes: 10 }; + const journal = await EmbeddingJobJournal.create(directory, identity, wideManifest); + await journal.begin({ ...unit(0), nodes: 64 }); + const ids = Array.from({ length: 64 }, (_, i) => `Function:a.ts:${i}${'x'.repeat(40_000)}`); + await journal.recordInputRejections(ids); + const reopened = await EmbeddingJobJournal.open(journal.directory, identity, wideManifest); + expect(Object.keys(reopened.snapshot().active!.inputFailures!)).toHaveLength(64); + expect((await fs.stat(path.join(journal.directory, 'job.json'))).size).toBeLessThan(32 * 1024); }); it.each([0, 4, 1.5])('rejects malformed persisted input failure count %s', async (count) => { diff --git a/gitnexus/test/unit/watch-supervisor-reservation.test.ts b/gitnexus/test/unit/watch-supervisor-reservation.test.ts index e2fa2f0b1..3f7589bae 100644 --- a/gitnexus/test/unit/watch-supervisor-reservation.test.ts +++ b/gitnexus/test/unit/watch-supervisor-reservation.test.ts @@ -64,18 +64,23 @@ it.each([undefined, String(320 * 1024 ** 2)])( }, ); -it('retains the parent reservation across watch refreshes', async () => { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); - directories.push(directory); - vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', String(320 * 1024 ** 2)); - vi.stubEnv('GITNEXUS_RC', ''); - const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; - await resolveWatchOptions(directory, {}, baseline); - await resolveWatchOptions(directory, {}, baseline); - expect(mocks.resources).toHaveBeenCalledTimes(2); - for (const [request] of mocks.resources.mock.calls) - expect(request).toMatchObject({ supervisorBytes: 320 * 1024 ** 2 }); -}); +it.each([undefined, String(320 * 1024 ** 2)])( + 'retains supervisor reservation %s across watch refreshes', + async (inherited) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); + directories.push(directory); + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', inherited); + vi.stubEnv('GITNEXUS_RC', ''); + const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; + await resolveWatchOptions(directory, {}, baseline); + await resolveWatchOptions(directory, {}, baseline); + expect(mocks.resources).toHaveBeenCalledTimes(2); + for (const [request] of mocks.resources.mock.calls) + expect(request).toMatchObject({ + supervisorBytes: inherited === undefined ? 192 * 1024 ** 2 : Number(inherited), + }); + }, +); it('honors GITNEXUS_MEMORY=off in the actual watch admission path', async () => { const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-manual-')); From 26d27823e4d0df8f87a5c4d286a365c9a754f4dc Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 17:56:21 +0000 Subject: [PATCH 53/67] perf: batch provider vector writes within units --- .../core/embeddings/embedding-job-runner.ts | 32 +++++++++++++++++-- .../embedding-vector-store.test.ts | 23 +++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts index f4f8a3eef..6206529c9 100644 --- a/gitnexus/src/core/embeddings/embedding-job-runner.ts +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -170,6 +170,8 @@ export async function runEmbeddingJob( const releaseScratch = budget.reserve(scratchBytes); let batch: Array & { text: string; release: () => void }> = []; + // Retained vector copies share one bounded write buffer, whether their + // source is a reusable artifact or a newly completed provider request. let copyBatch: VectorWriteChunk[] = []; let copyBytes = 0; let copyReleases: Array<() => void> = []; @@ -196,6 +198,30 @@ export async function runEmbeddingJob( } }; + const queueWrites = async (chunks: VectorWriteChunk[]): Promise => { + const bytes = chunks.reduce( + (total, chunk) => total + 32 * dimensions + 1024 + 4 * chunk.nodeId.length, + 0, + ); + if ( + copyBatch.length && + (copyBatch.length + chunks.length > 256 || + copyBytes + bytes > store.options.batchBytes || + budget.usedBytes + bytes > budget.limit) + ) + await flushCopies(); + // The active provider batch already reserves these vector copies. A + // tight queue can write immediately without a second retained reservation. + if (bytes > store.options.batchBytes || budget.usedBytes + bytes > budget.limit) { + await store.appendBatch(chunks); + return; + } + const release = budget.reserve(bytes); + copyBatch.push(...chunks); + copyBytes += bytes; + copyReleases.push(release); + }; + const flush = async (): Promise => { if (!batch.length) return; signal?.throwIfAborted(); @@ -250,7 +276,7 @@ export async function runEmbeddingJob( }, ], ); - if (updates.length) await store.appendBatch(updates); + if (updates.length) await queueWrites(updates); } if (providerFailed) failures++; if (!abort && providerFailed && attempted >= ratioFloor && failures / attempted >= 0.25) @@ -445,7 +471,6 @@ export async function runEmbeddingJob( reusedNodes++; continue; } - if (copyBatch.length) await flushCopies(); const state = { contentHash: input.contentHash, chunks: 0, @@ -466,6 +491,7 @@ export async function runEmbeddingJob( ) await flush(); if (failed.has(node.id) || abort) break; + if (copyBatch.length && budget.usedBytes + bytes > budget.limit) await flushCopies(); const release = budget.reserve(bytes); batch.push({ ...input, @@ -478,8 +504,8 @@ export async function runEmbeddingJob( if (state.ended) state.inputsDigest = hash?.digest('hex'); if (abort) break; } - if (copyBatch.length) await flushCopies(); if (!abort) await flush(); + if (copyBatch.length) await flushCopies(); const attempts = await journal.recordInputRejections([...rejectedInputs]); for (const id of rejectedInputs) { if (attempts[id] < 3) continue; diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index f3808ca7f..d052dcf36 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -247,6 +247,24 @@ describe('rebased pending vector inputs', () => { }); describe('bounded durable embedding runner', () => { + it.each([256, 512])( + 'retains generated writes within a %i KiB queue until durable completion', + async (queueKiB) => { + const fixture = await runnerFixture(64); + const append = vi.spyOn(store, 'appendBatch'); + const result = await fixture.run({ queueBytes: queueKiB * 1024 }); + expect(result.state).toMatchObject({ completedNodes: 64, chunks: 64, failedUnits: 0 }); + expect(result.peakQueueBytes).toBeLessThanOrEqual(queueKiB * 1024); + expect(append.mock.calls.flatMap(([rows]) => rows)).toHaveLength(64); + if (queueKiB === 512) + expect(append.mock.calls.length).toBeLessThan(fixture.provider.mock.calls.length); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).state.chunks).toBe(64); + expect(fixture.provider).not.toHaveBeenCalled(); + }, + ); + it('completes copied groups together at the unit boundary without provider requests', async () => { const fixture = await runnerFixture(64); const finish = vi.spyOn(store, 'finishNodes'); @@ -271,7 +289,7 @@ describe('bounded durable embedding runner', () => { expect(queries.mock.calls.filter(([sql]) => sql === 'COMMIT').length).toBeLessThan(10); }); - it('flushes copied vectors before generated inputs and preserves complete mixed coverage', async () => { + it('bounds copied and generated vectors together and preserves complete mixed coverage', async () => { const fixture = await runnerFixture(65); const result = await fixture.run({ queueBytes: 512 * 1024, @@ -517,9 +535,8 @@ describe('bounded durable embedding runner', () => { async (when) => { const fixture = await runnerFixture(65); const append = store.appendBatch.bind(store); - let calls = 0; vi.spyOn(store, 'appendBatch').mockImplementation(async (rows) => { - if (++calls === 9) { + if (rows.some((row) => row.nodeId === 'Const:source.ts:n00064')) { if (when === 'after') await append(rows); throw new Error('simulated write failure'); } From 01c27de210c39acd4d068687cf2c5613716cdfed Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 18:10:19 +0000 Subject: [PATCH 54/67] fix: harden cleanup and semantic availability --- MIGRATION.md | 6 +++++ gitnexus/src/cli/clean.ts | 5 ++-- .../src/core/lbug/legacy-vector-export.ts | 8 ++++++ gitnexus/src/server/api.ts | 4 +-- gitnexus/src/storage/embedding-artifact.ts | 27 ++++++++++++------- .../src/storage/legacy-embedding-source.ts | 13 ++++++--- gitnexus/src/storage/stale-branch-slots.ts | 15 +++++------ .../legacy-embedding-controller.test.ts | 14 ++++++++++ .../integration/legacy-vector-export.test.ts | 6 ++--- gitnexus/test/unit/api-fts-mode.test.ts | 21 ++++++++++++++- .../test/unit/clean-command-ownership.test.ts | 24 +++++++++++++++++ gitnexus/test/unit/embedding-artifact.test.ts | 15 +++++++++++ .../test/unit/legacy-embedding-source.test.ts | 24 ++++++++++++++++- 13 files changed, 151 insertions(+), 31 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 62481cba2..f979f2658 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -31,6 +31,12 @@ three deterministic input rejections, the job publishes healthy nodes and record On Windows, a legacy reader can prevent retirement of `lbug`. Stop that reader and rerun analyze; the prepared publication and vector artifacts are retained. +Legacy source receipts now include the file's change time as well as its size +and modification time. Receipts from earlier previews without that guard are +retained but cannot certify reuse. Keep them with the original source when +recovering from a backup or re-exporting a verified source; do not fill in a +missing guard from the current file, which may have changed since retirement. + `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` is no longer accepted by analysis: it asks the native engine to allocate an unbounded pool. Remove this setting from CLI, server and auto-sync environments to use automatic sizing, or set a positive diff --git a/gitnexus/src/cli/clean.ts b/gitnexus/src/cli/clean.ts index 1ac98cdf4..2913b6e14 100644 --- a/gitnexus/src/cli/clean.ts +++ b/gitnexus/src/cli/clean.ts @@ -34,6 +34,7 @@ import { } from '../core/lbug/sidecar-recovery.js'; import { t } from './i18n/index.js'; import { getGlobalDir } from '../storage/global-dir.js'; +import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js'; import { STORES_DIR } from '../storage/shared-store.js'; import { findLegacyLocalIndex, @@ -271,8 +272,8 @@ export const cleanCommand = async (options?: { console.log(t('clean.branchNotIndexed', { branch: options.branch })); return; } - const { lbugPath } = getStoragePaths(repo.repoPath, summary.branch, storagePath); - const branchDir = path.dirname(lbugPath); + // Cleanup targets the owned slot, even when its graph pointer is damaged. + const branchDir = path.join(storagePath, BRANCHES_DIR, branchSlug(summary.branch)); if (!isContainedBranchDir(storagePath, branchDir)) { logger.error( `Refusing to clean branch index outside the validated storage slot: ${branchDir}`, diff --git a/gitnexus/src/core/lbug/legacy-vector-export.ts b/gitnexus/src/core/lbug/legacy-vector-export.ts index 52db9bd4a..1a09b5ec6 100644 --- a/gitnexus/src/core/lbug/legacy-vector-export.ts +++ b/gitnexus/src/core/lbug/legacy-vector-export.ts @@ -118,12 +118,14 @@ export async function exportLegacyVectors( const sourceFile = `lbug.generation.${id.replaceAll('-', '')}`; const destination = path.join(root, sourceFile); const csvLiteral = `'${escapeCypherString(csv)}'`; + let stagingReleased = true; try { options.signal?.throwIfAborted(); await options.query( `COPY (MATCH (e:CodeEmbedding) RETURN ${fields.map((f) => `e.${f} AS ${f}`).join(', ')}) TO ${csvLiteral} (HEADER=true)`, ); options.signal?.throwIfAborted(); + stagingReleased = false; const handle = await openLbugConnection(lbug, staging, { bufferPoolBytes: options.bufferPoolBytes, threads: options.threads, @@ -154,6 +156,7 @@ export async function exportLegacyVectors( // A failed physical release remains charged by the native reservation // manager. Never rename or certify that still-open export. await closeLbugConnection(handle, { strict: true, databasePath: staging }); + stagingReleased = true; } options.signal?.throwIfAborted(); await assertLbugPublicationReady(staging); @@ -173,5 +176,10 @@ export async function exportLegacyVectors( ); } finally { await fs.rm(csv, { force: true }); + // The original source remains untouched. A failed close must retain its + // live file; renamed generations are reclaimed by the ownership sweep, + // since a receipt write can succeed before reporting a durability failure. + if (stagingReleased) + for (const suffix of ['', '.wal', '.shadow']) await fs.rm(staging + suffix, { force: true }); } } diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index bf5864ea6..eb91acbf8 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -1526,9 +1526,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => ? { metadataDirectory: entry.storagePath, metadata, graphPath: lbugPath } : undefined; const semanticReady = - mode !== 'bm25' && - (snapshot !== undefined || - (await import('../core/embeddings/embedder.js')).isEmbedderReady()); + mode !== 'bm25' && (await import('../core/embeddings/embedder.js')).isEmbedderReady(); const semanticLane = async (execute: typeof executeQuery, text: string, count = 10) => { const { semanticSearch } = await import('../core/embeddings/embedding-pipeline.js'); return semanticSearch( diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts index 3e4dea315..47f30c25c 100644 --- a/gitnexus/src/storage/embedding-artifact.ts +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -74,7 +74,7 @@ export async function readEmbeddingArtifactOwners(root: string): Promise { - const directory = await embeddingJobDirectory(root); - return createEmbeddingJobDirectories(directory, [ - 'embeddings', - 'spaces', - embeddingArtifactSpaceKey(space), - ]); +async function spaceDirectory( + root: string, + space: EmbeddingArtifactSpace, + create = true, +): Promise { + let directory = await embeddingJobDirectory(root); + const parts = ['embeddings', 'spaces', embeddingArtifactSpaceKey(space)]; + if (create) return createEmbeddingJobDirectories(directory, parts); + for (const part of parts) directory = await embeddingJobDirectory(path.join(directory, part)); + return directory; } async function optionalRecord(directory: string, name: string): Promise { @@ -240,7 +243,13 @@ export async function readEmbeddingArtifactSpace( space: EmbeddingArtifactSpace, validateOwner: ValidateOwner, ): Promise { - const directory = await spaceDirectory(root, space); + let directory: string; + try { + directory = await spaceDirectory(root, space, false); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } const currentRaw = await optionalRecord(directory, 'space.json'); let current = currentRaw === null ? null : record(currentRaw, space); const intent = (await optionalRecord(directory, 'rebase.json')) as { diff --git a/gitnexus/src/storage/legacy-embedding-source.ts b/gitnexus/src/storage/legacy-embedding-source.ts index 98cf9c803..f4f0c1574 100644 --- a/gitnexus/src/storage/legacy-embedding-source.ts +++ b/gitnexus/src/storage/legacy-embedding-source.ts @@ -33,7 +33,7 @@ export interface LegacyEmbeddingSourceRecord { recordedInputIdentity?: string; /** Descriptor attributes detect replacement/mutation. Ownership and provider * provenance come from the retired slot and record, never these attributes. */ - file: { dev: string; ino: string; size: string; mtimeNs: string }; + file: { dev: string; ino: string; size: string; mtimeNs: string; ctimeNs: string }; excluded: { digest: string; bytes: number; nodes: number }; } export interface ResolvedLegacyEmbeddingSource extends LegacyEmbeddingSourceRecord { @@ -73,7 +73,7 @@ function record(value: unknown): LegacyEmbeddingSourceRecord { r.schemaFingerprint.length > 2048 || (r.recordedInputIdentity !== undefined && !DIGEST.test(r.recordedInputIdentity)) || !r.file || - ![r.file.dev, r.file.ino, r.file.size, r.file.mtimeNs].every( + ![r.file.dev, r.file.ino, r.file.size, r.file.mtimeNs, r.file.ctimeNs].every( (v) => typeof v === 'string' && /^\d+$/.test(v), ) || !r.excluded || @@ -91,7 +91,13 @@ function record(value: unknown): LegacyEmbeddingSourceRecord { ...(r.recordedInputIdentity === undefined ? {} : { recordedInputIdentity: r.recordedInputIdentity }), - file: { dev: r.file.dev, ino: r.file.ino, size: r.file.size, mtimeNs: r.file.mtimeNs }, + file: { + dev: r.file.dev, + ino: r.file.ino, + size: r.file.size, + mtimeNs: r.file.mtimeNs, + ctimeNs: r.file.ctimeNs, + }, excluded: { digest: r.excluded.digest, bytes: r.excluded.bytes, nodes: r.excluded.nodes }, }; } @@ -103,6 +109,7 @@ function fileIdentity(stat: BigIntStats): LegacyEmbeddingSourceRecord['file'] { ino: String(stat.ino), size: String(stat.size), mtimeNs: String(stat.mtimeNs), + ctimeNs: String(stat.ctimeNs), }; } async function sourceFile(root: string, r: LegacyEmbeddingSourceRecord): Promise { diff --git a/gitnexus/src/storage/stale-branch-slots.ts b/gitnexus/src/storage/stale-branch-slots.ts index aee92c0ed..49d111f4b 100644 --- a/gitnexus/src/storage/stale-branch-slots.ts +++ b/gitnexus/src/storage/stale-branch-slots.ts @@ -9,11 +9,11 @@ import fs from 'fs/promises'; import path from 'path'; -import { BRANCHES_DIR } from './branch-index.js'; +import { BRANCHES_DIR, branchSlug } from './branch-index.js'; import { listLocalHeads } from './git.js'; import { isMissingFilesystemError, loadMeta } from './repo-meta.js'; import { mapPool } from './map-pool.js'; -import { getStoragePaths, removeBranchIndex } from './repo-manager.js'; +import { removeBranchIndex } from './repo-manager.js'; export type StaleBranchReason = | 'ref-missing' @@ -40,8 +40,8 @@ export interface ListStaleBranchSlotsInput { includeSize?: boolean; } -const slotDirForBranch = (repoPath: string, storagePath: string, branch: string): string => - path.dirname(getStoragePaths(repoPath, branch, storagePath).metaPath); +const slotDirForBranch = (storagePath: string, branch: string): string => + path.join(storagePath, BRANCHES_DIR, branchSlug(branch)); /** Same bound as `listRegisteredRepos`: cap concurrent slot I/O. */ const STALE_SLOT_IO_CONCURRENCY = 8; @@ -191,10 +191,7 @@ export const listStaleBranchSlots = async ( const registryByDir = new Map(); for (const row of recorded) { - registryByDir.set( - path.resolve(slotDirForBranch(input.repoPath, input.storagePath, row.branch)), - row.branch, - ); + registryByDir.set(path.resolve(slotDirForBranch(input.storagePath, row.branch)), row.branch); } const branchesProbe = await probeContainedBranchesRoot(input.storagePath); @@ -519,7 +516,7 @@ export const removeBranchSlot = async ( } const dropRegistry = - dir === null || isSameNormalizedPath(dir, slotDirForBranch(repoPath, storagePath, branch)); + dir === null || isSameNormalizedPath(dir, slotDirForBranch(storagePath, branch)); if (dropRegistry) { try { await removeBranchIndex(repoPath, branch); diff --git a/gitnexus/test/integration/legacy-embedding-controller.test.ts b/gitnexus/test/integration/legacy-embedding-controller.test.ts index 31fe43a0d..ebc0c0040 100644 --- a/gitnexus/test/integration/legacy-embedding-controller.test.ts +++ b/gitnexus/test/integration/legacy-embedding-controller.test.ts @@ -305,6 +305,20 @@ describe('legacy migration through the published job controller', () => { const graph = f.options().graphPath; await fs.unlink(graph); await fs.link(f.sourcePath, graph); + // Refresh the fixture receipt after link creation changes ctime, so this + // still exercises the separate live-graph alias guard. + const legacyVectorSource = await recordLegacyEmbeddingSource( + f.root, + { + sourceFile: path.basename(f.sourcePath), + schemaFingerprint: 'schema', + recordedIdentity: { provider: 'local', model: 'test', dimensions: 4 }, + }, + [], + ); + const metadata = await loadMeta(f.root); + if (!metadata) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { ...metadata, legacyVectorSource }); await expect(f.run()).rejects.toThrow('distinct from the mutable graph'); expect(f.provider).not.toHaveBeenCalled(); }); diff --git a/gitnexus/test/integration/legacy-vector-export.test.ts b/gitnexus/test/integration/legacy-vector-export.test.ts index cf216b2f5..02ece741a 100644 --- a/gitnexus/test/integration/legacy-vector-export.test.ts +++ b/gitnexus/test/integration/legacy-vector-export.test.ts @@ -230,9 +230,9 @@ it('refuses publication when the source count changes during the transfer', asyn }, }), ).rejects.toThrow('source changed'); - expect( - (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), - ).toEqual([]); + expect((await fs.readdir(f.root)).filter((name) => /generation|staging/.test(name))).toEqual( + [], + ); expect((await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count'))[0].count).toBe(1); } finally { await f.close(); diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 32645f982..823f0978c 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -30,6 +30,7 @@ const mocks = vi.hoisted(() => ({ { nodeId: 'Function:a.ts:alpha', name: 'alpha', distance: 0 }, ]), embedQuery: vi.fn(async () => [1, 0, 0, 0]), + isEmbedderReady: vi.fn(() => false), runAnalysisEmbeddingPhase: vi.fn(), withLbugDb: vi.fn(), search: vi.fn(), @@ -58,7 +59,9 @@ vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline: mocks.runEmbeddingPipeline, semanticSearch: mocks.semanticSearch, })); -vi.mock('../../src/core/embeddings/embedder.js', () => ({ isEmbedderReady: () => false })); +vi.mock('../../src/core/embeddings/embedder.js', () => ({ + isEmbedderReady: mocks.isEmbedderReady, +})); vi.mock('../../src/mcp/core/embedder.js', () => ({ embedQuery: mocks.embedQuery })); vi.mock('../../src/core/analysis-embedding-phase.js', () => ({ runAnalysisEmbeddingPhase: mocks.runAnalysisEmbeddingPhase, @@ -170,6 +173,7 @@ afterAll(() => { beforeEach(() => { vi.clearAllMocks(); + mocks.isEmbedderReady.mockReturnValue(false); mocks.activeJob = undefined; mocks.acquireIndexLock.mockResolvedValue({ release: mocks.releaseIndexLock, @@ -657,6 +661,7 @@ describe('GET /api/repos catalog validation', () => { it.each(['semantic', 'hybrid'])( 'searches published vectors in %s mode without requiring a previously loaded model', async (mode) => { + mocks.isEmbedderReady.mockReturnValue(true); const metadata = { graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, }; @@ -675,6 +680,20 @@ it.each(['semantic', 'hybrid'])( }, ); +it.each(['semantic', 'hybrid'])( + 'degrades %s search when published vectors have no usable embedder', + async (mode) => { + mocks.loadMeta.mockResolvedValue({ + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + }); + const response = await invoke('/api/search', {}, { mode }); + expect(response.body.results).toEqual([]); + expect(mocks.semanticSearch).not.toHaveBeenCalled(); + expect(mocks.embedQuery).not.toHaveBeenCalled(); + expect(mocks.search).toHaveBeenCalledTimes(mode === 'hybrid' ? 1 : 0); + }, +); + it('waits for embedding native cleanup before closing the backend or exiting on shutdown', async () => { mocks.loadMeta.mockResolvedValue({ repoPath: entry.path }); let finish!: () => void; diff --git a/gitnexus/test/unit/clean-command-ownership.test.ts b/gitnexus/test/unit/clean-command-ownership.test.ts index de682fe89..70651a91e 100644 --- a/gitnexus/test/unit/clean-command-ownership.test.ts +++ b/gitnexus/test/unit/clean-command-ownership.test.ts @@ -173,4 +173,28 @@ describe('cleanCommand named branch empty-dir cleanup (#3331)', () => { await expect(fs.access(path.join(storagePath, 'branches'))).rejects.toBeTruthy(); await expect(fs.access(path.join(storagePath, 'gitnexus.json'))).resolves.toBeUndefined(); }); + + it.each(['branch', 'stale'])( + 'cleans a damaged generation pointer through %s without following it', + async (mode) => { + const { getStoragePaths, saveMeta } = await import('../../src/storage/repo-manager.js'); + const branchDir = path.dirname(getStoragePaths(repo, 'feature/x', storagePath).metaPath); + const outside = path.join(fixture.dbPath, 'external-sentinel'); + await fs.writeFile(outside, 'keep external data'); + const graphPath = 'lbug.generation.' + 'a'.repeat(32); + await fs.symlink(outside, path.join(branchDir, graphPath)); + await saveMeta(branchDir, { + repoPath: repo, + lastCommit: 'aaa', + indexedAt: '2026-09-20T00:00:00.000Z', + branch: 'feature/x', + graphPath, + }); + await cleanCommand( + mode === 'branch' ? { branch: 'feature/x', force: true } : { stale: true, force: true }, + ); + await expect(fs.access(branchDir)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(outside, 'utf8')).toBe('keep external data'); + }, + ); }); diff --git a/gitnexus/test/unit/embedding-artifact.test.ts b/gitnexus/test/unit/embedding-artifact.test.ts index e65548129..188dfb85c 100644 --- a/gitnexus/test/unit/embedding-artifact.test.ts +++ b/gitnexus/test/unit/embedding-artifact.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { adoptEmbeddingArtifact, readEmbeddingArtifactSpace, + readEmbeddingArtifactOwners, embeddingArtifactSpaceKey, type EmbeddingArtifactOwner, type EmbeddingArtifactRecord, @@ -39,6 +40,20 @@ afterEach(async () => { }); describe('pending embedding artifact ownership', () => { + it('does not create ownership directories for a missing space', async () => { + expect(await readEmbeddingArtifactSpace(root, space, validate)).toBeNull(); + expect(await fs.readdir(root)).toEqual([]); + expect(validate).not.toHaveBeenCalled(); + }); + + it('ignores an empty interrupted space but retains unidentified contents', async () => { + await fs.mkdir(directory(), { recursive: true }); + expect(await readEmbeddingArtifactOwners(root)).toEqual([]); + await fs.writeFile(path.join(directory(), 'unknown'), 'retain'); + await expect(readEmbeddingArtifactOwners(root)).rejects.toThrow('no ownership receipt'); + expect(await fs.readFile(path.join(directory(), 'unknown'), 'utf8')).toBe('retain'); + }); + it('keeps one artifact through repeated source generations without enumerating job history', async () => { vi.spyOn(fs, 'readdir').mockRejectedValue(new Error('History scan')); vi.spyOn(fs, 'opendir').mockRejectedValue(new Error('History scan')); diff --git a/gitnexus/test/unit/legacy-embedding-source.test.ts b/gitnexus/test/unit/legacy-embedding-source.test.ts index cf3599816..a427ce605 100644 --- a/gitnexus/test/unit/legacy-embedding-source.test.ts +++ b/gitnexus/test/unit/legacy-embedding-source.test.ts @@ -154,8 +154,30 @@ describe('retained legacy source receipts', () => { await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('regular file'); await fs.unlink(f.source); await fs.rename(f.source + '.old', f.source); + const restoredIdentity = await recordLegacyEmbeddingSource(f.root, f.input, []); await fs.writeFile(f.source + '.wal', 'not checkpointed'); - await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('native recovery'); + await expect(readLegacyEmbeddingSource(f.root, restoredIdentity)).rejects.toThrow( + 'native recovery', + ); + }); + it('rejects in-place source changes even when size and mtime are restored', async () => { + const f = await fixture(); + const timestamp = new Date('2020-01-01T00:00:00Z'); + await fs.utimes(f.source, timestamp, timestamp); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + const before = await fs.stat(f.source, { bigint: true }); + await fs.writeFile(f.source, 'changed database'); + await fs.utimes(f.source, timestamp, timestamp); + const after = await fs.stat(f.source, { bigint: true }); + expect({ ino: after.ino, size: after.size, mtime: after.mtimeNs }).toEqual({ + ino: before.ino, + size: before.size, + mtime: before.mtimeNs, + }); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow( + 'changed after retirement', + ); + expect(await fs.readFile(f.source, 'utf8')).toBe('changed database'); }); it('validates the complete exclusion checksum before a native index may become complete', async () => { const f = await fixture(); From a1feae03ae59aa797c35b804106166e7509036ed Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 18:25:44 +0000 Subject: [PATCH 55/67] perf: avoid repeated analysis startup probes Skip unused automatic-heap parent provenance and reuse one npm version probe for runner selection and its warning. Keep child/watch identity capture and manual/test heap overrides. Refs #3829. --- gitnexus/src/cli/index.ts | 14 ++++++++ gitnexus/src/cli/resolve-invocation.ts | 14 ++++++-- gitnexus/test/unit/analyze-bootstrap.test.ts | 33 +++++++++++++++++-- gitnexus/test/unit/resolve-invocation.test.ts | 19 +++++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 6168dc9ff..1d750fc9a 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -17,6 +17,7 @@ import { localizeCliHelp } from './help-i18n.js'; import { t } from './i18n/index.js'; import { writeCommandBanner } from './command-banner.js'; import { runProcessCliUpdateNotice } from './update-notice.js'; +import { memoryAutopilotDisabled } from '../core/ingestion/utils/effective-ram.js'; const program = new Command(); @@ -302,8 +303,21 @@ program !options.watch && options.memoryBudget !== undefined && process.env.GITNEXUS_HEAP_LIMIT_SOURCE !== 'budget'; + // An unconfigured automatic-heap parent also always re-execs with the + // admitted old/semi-space flags. Keep conservative capture for custom Node + // flags, manual heaps, test overrides and every analyzing child. + const autoSupervisor = + !options.watch && + options.memoryBudget === undefined && + process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && + process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP !== '1' && + !memoryAutopilotDisabled() && + process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB === undefined && + process.execArgv.length === 0 && + !(process.env.NODE_OPTIONS ?? '').trim(); if ( budgetSupervisor || + autoSupervisor || (process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected) ) { await createLbugLazyAction(() => import('./analyze.js'), 'analyzeCommand')( diff --git a/gitnexus/src/cli/resolve-invocation.ts b/gitnexus/src/cli/resolve-invocation.ts index a371985c5..145b3a98a 100644 --- a/gitnexus/src/cli/resolve-invocation.ts +++ b/gitnexus/src/cli/resolve-invocation.ts @@ -22,6 +22,7 @@ interface InvocationResolver { // no argument. resolveInvocationMode: ( probe?: (command: string, gitnexusWrapper?: boolean) => string | null, + deps?: { readonly npmMajor?: number | null }, ) => InvocationMode; formatDocumentationDlxCommand: ( gitnexusArgs: string, @@ -91,8 +92,17 @@ export function getNpmMajorVersion(): number | null { * cannot run. */ export function warnIfNpm11NpxRisk(): void { - if (resolveInvocationMode() !== 'npx') return; - const major = getNpmMajorVersion(); + // The resolver and warning need the same version. Probe once, lazily: a + // preferred global binary or forced alternative still needs no npm spawn. + let cachedMajor: number | null | undefined; + const deps = { + get npmMajor(): number | null { + if (cachedMajor === undefined) cachedMajor = getNpmMajorVersion(); + return cachedMajor; + }, + }; + if (resolveInvocationMode(undefined, deps) !== 'npx') return; + const major = deps.npmMajor; if (major === null || major < 11) return; process.stderr.write( `Warning: npm ${major}.x can crash while installing gitnexus via npx ` + diff --git a/gitnexus/test/unit/analyze-bootstrap.test.ts b/gitnexus/test/unit/analyze-bootstrap.test.ts index c24f32b17..b38b7c3ce 100644 --- a/gitnexus/test/unit/analyze-bootstrap.test.ts +++ b/gitnexus/test/unit/analyze-bootstrap.test.ts @@ -21,12 +21,22 @@ vi.mock('../../src/cli/analyze.js', () => ({ vi.mock('../../src/cli/update-notice.js', () => ({ runProcessCliUpdateNotice: vi.fn() })); const originalArgv = process.argv; +const originalExecArgv = process.execArgv; const connected = Object.getOwnPropertyDescriptor(process, 'connected'); const receipt = { schemaVersion: 4 }; beforeEach(() => { vi.resetModules(); vi.clearAllMocks(); + process.execArgv = []; + for (const key of [ + 'GITNEXUS_ANALYSIS_SUPERVISED', + 'GITNEXUS_ANALYSIS_MANUAL_HEAP', + 'GITNEXUS_MEMORY', + 'GITNEXUS_TEST_RESPAWN_HEAP_MB', + 'NODE_OPTIONS', + ]) + vi.stubEnv(key, undefined); mocks.capture.mockImplementation(async (_url, loader) => ({ runnerIdentity: receipt, loaded: await loader(), @@ -39,23 +49,40 @@ beforeEach(() => { afterEach(() => { process.argv = originalArgv; + process.execArgv = originalExecArgv; if (connected) Object.defineProperty(process, 'connected', connected); else delete (process as Partial).connected; vi.unstubAllEnvs(); vi.restoreAllMocks(); }); -it.each([ - { continuation: false, watch: false, budget: false, source: '', captures: true }, +it.each<{ + continuation?: boolean; + watch?: boolean; + budget?: boolean; + source?: string; + env?: Record; + execArgv?: string[]; + captures: boolean; +}>([ + { continuation: false, watch: false, budget: false, source: '', captures: false }, { continuation: false, watch: true, budget: false, source: '', captures: true }, { continuation: true, watch: false, budget: false, source: '', captures: false }, { continuation: false, watch: false, budget: true, source: '', captures: false }, { continuation: false, watch: false, budget: true, source: 'auto', captures: false }, { continuation: false, watch: false, budget: true, source: 'budget', captures: true }, { continuation: false, watch: true, budget: true, source: '', captures: true }, + { env: { GITNEXUS_ANALYSIS_SUPERVISED: '1' }, captures: true }, + { env: { GITNEXUS_ANALYSIS_MANUAL_HEAP: '1' }, captures: true }, + { env: { GITNEXUS_MEMORY: 'off' }, captures: true }, + { env: { GITNEXUS_TEST_RESPAWN_HEAP_MB: '128' }, captures: true }, + { env: { NODE_OPTIONS: '--max_old_space_size=2048' }, captures: true }, + { execArgv: ['--max-old-space-size=2048'], captures: true }, ])('selects the analyzer bootstrap for %j', async (scenario) => { vi.stubEnv('GITNEXUS_ANALYSIS_CONTINUATION', scenario.continuation ? '1' : ''); - vi.stubEnv('GITNEXUS_HEAP_LIMIT_SOURCE', scenario.source); + vi.stubEnv('GITNEXUS_HEAP_LIMIT_SOURCE', scenario.source ?? ''); + for (const [key, value] of Object.entries(scenario.env ?? {})) vi.stubEnv(key, value); + if (scenario.execArgv) process.execArgv = scenario.execArgv; process.argv = [process.execPath, 'gitnexus', 'analyze', '/fixture']; if (scenario.watch) process.argv.push('--watch'); if (scenario.budget) process.argv.push('--memory-budget', '2048'); diff --git a/gitnexus/test/unit/resolve-invocation.test.ts b/gitnexus/test/unit/resolve-invocation.test.ts index 5d17736a6..bf835a525 100644 --- a/gitnexus/test/unit/resolve-invocation.test.ts +++ b/gitnexus/test/unit/resolve-invocation.test.ts @@ -376,6 +376,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { describe('warnIfNpm11NpxRisk (#1939 npm-11 nudge)', () => { afterEach(() => { vi.clearAllMocks(); + vi.unstubAllEnvs(); delete process.env.GITNEXUS_INVOCATION; }); @@ -452,12 +453,30 @@ describe('warnIfNpm11NpxRisk (#1939 npm-11 nudge)', () => { write.mockRestore(); }); + it.each(['11.0.0\n', '10.9.0\n', 'unreadable\n'])( + 'shares one npm probe with automatic runner selection for %s', + (version) => { + vi.stubEnv('GITNEXUS_INVOCATION', ''); + vi.stubEnv('PATH', ''); + mockedExec.mockReturnValue(version); + const write = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + try { + warnIfNpm11NpxRisk(); + expect(mockedExec).toHaveBeenCalledOnce(); + expect(write).toHaveBeenCalledTimes(version.startsWith('11.') ? 1 : 0); + } finally { + write.mockRestore(); + } + }, + ); + it('does not warn when a global gitnexus or pnpm is preferred', () => { process.env.GITNEXUS_INVOCATION = 'pnpm'; mockedExec.mockReturnValue('11.0.0\n'); const write = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); warnIfNpm11NpxRisk(); expect(write).not.toHaveBeenCalled(); + expect(mockedExec).not.toHaveBeenCalled(); write.mockRestore(); }); From 92289b6e9bdc2dee5d318fda553d2218ddef5122 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 18:36:46 +0000 Subject: [PATCH 56/67] fix: normalize legacy export paths on Windows Cover Unicode export directories and restore the heap parser comment. Refs #3829. --- gitnexus/src/cli/analyze.ts | 10 +- .../src/core/lbug/legacy-vector-export.ts | 4 +- .../integration/legacy-vector-export.test.ts | 127 +++++++++--------- 3 files changed, 72 insertions(+), 69 deletions(-) diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 9ae5eda6c..4c0687dc7 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -605,11 +605,6 @@ const forceHeapOOMForTestIfEnabled = (): void => { // `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value. const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024; -/** - * Last `--max-old-space-size` value (MB) in a NODE_OPTIONS string, or `null` - * when absent/unparseable. Last occurrence wins, matching V8's own - * later-flag-wins semantics when NODE_OPTIONS repeats a flag. - */ /** CLI and watch must preserve the same explicit heap/autopilot opt-outs. */ export function usesManualAnalysisHeap(memoryBudget?: string): boolean { return ( @@ -622,6 +617,11 @@ export function usesManualAnalysisHeap(memoryBudget?: string): boolean { ); } +/** + * Last `--max-old-space-size` value (MB) in a NODE_OPTIONS string, or `null` + * when absent/unparseable. Last occurrence wins, matching V8's own + * later-flag-wins semantics when NODE_OPTIONS repeats a flag. + */ export function parseMaxOldSpaceMb(nodeOptions: string): number | null { return parseV8SizeFlagMb(nodeOptions, 'max-old-space-size'); } diff --git a/gitnexus/src/core/lbug/legacy-vector-export.ts b/gitnexus/src/core/lbug/legacy-vector-export.ts index 1a09b5ec6..8f6744d32 100644 --- a/gitnexus/src/core/lbug/legacy-vector-export.ts +++ b/gitnexus/src/core/lbug/legacy-vector-export.ts @@ -3,7 +3,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; import lbug from '@ladybugdb/core'; -import { openLbugConnection, closeLbugConnection } from './lbug-config.js'; +import { openLbugConnection, closeLbugConnection, toNativeSafePath } from './lbug-config.js'; import { escapeCypherString } from './cypher-escape.js'; import { assertLbugPublicationReady } from './publication.js'; import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; @@ -117,7 +117,7 @@ export async function exportLegacyVectors( const csv = staging + '.csv'; const sourceFile = `lbug.generation.${id.replaceAll('-', '')}`; const destination = path.join(root, sourceFile); - const csvLiteral = `'${escapeCypherString(csv)}'`; + const csvLiteral = `'${escapeCypherString(toNativeSafePath(csv).replace(/\\/g, '/'))}'`; let stagingReleased = true; try { options.signal?.throwIfAborted(); diff --git a/gitnexus/test/integration/legacy-vector-export.test.ts b/gitnexus/test/integration/legacy-vector-export.test.ts index 02ece741a..db3a6e2f3 100644 --- a/gitnexus/test/integration/legacy-vector-export.test.ts +++ b/gitnexus/test/integration/legacy-vector-export.test.ts @@ -30,8 +30,8 @@ afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); }); -async function fixture(rows = 1) { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "gn-legacy-export-'")); +async function fixture(rows = 1, prefix = "gn-legacy-export-'") { + const root = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); roots.push(root); const file = path.join(root, 'lbug'); const handle = await openLbugConnection(lbug, file, { bufferPoolBytes: pool, threads: 1 }); @@ -75,73 +75,76 @@ async function fixture(rows = 1) { }; } -it('exports only the vector table and retains a compatible complete group without provider work', async () => { - const f = await fixture(); - const before = await fs.stat(f.file, { bigint: true }); - const copy = vi.spyOn(fs, 'copyFile'); - let store: VectorStore | undefined, source: LegacyVectorSource | undefined; - try { - const id = await exportLegacyVectors(f.options); - expect(id).toMatch(/^[a-f0-9]{64}$/); - const receipt = await readLegacyEmbeddingSource(f.root, id!); - expect(receipt.recordedIdentity).toEqual(identity); - expect(copy).not.toHaveBeenCalled(); - expect((await fs.stat(f.file, { bigint: true })).ino).toBe(before.ino); - expect(await f.query('MATCH (n:File) RETURN n.content AS content')).toEqual([ - { content: 'Graph content must stay out of the export' }, - ]); - const exported = await openLbugConnection(lbug, receipt.path, { - readOnly: true, - bufferPoolBytes: pool, - threads: 1, - }); +it.each(["gn-legacy-export-'", "gn-legacy-export-路径-'"])( + 'exports a reusable vector table under %s without provider work', + async (prefix) => { + const f = await fixture(1, prefix); + const before = await fs.stat(f.file, { bigint: true }); + const copy = vi.spyOn(fs, 'copyFile'); + let store: VectorStore | undefined, source: LegacyVectorSource | undefined; try { - const result = await exported.conn.query('CALL show_tables() RETURN name'); - const cursor = Array.isArray(result) ? result[0] : result; + const id = await exportLegacyVectors(f.options); + expect(id).toMatch(/^[a-f0-9]{64}$/); + const receipt = await readLegacyEmbeddingSource(f.root, id!); + expect(receipt.recordedIdentity).toEqual(identity); + expect(copy).not.toHaveBeenCalled(); + expect((await fs.stat(f.file, { bigint: true })).ino).toBe(before.ino); + expect(await f.query('MATCH (n:File) RETURN n.content AS content')).toEqual([ + { content: 'Graph content must stay out of the export' }, + ]); + const exported = await openLbugConnection(lbug, receipt.path, { + readOnly: true, + bufferPoolBytes: pool, + threads: 1, + }); try { - expect(await cursor.getAll()).toEqual([{ name: 'CodeEmbedding' }]); + const result = await exported.conn.query('CALL show_tables() RETURN name'); + const cursor = Array.isArray(result) ? result[0] : result; + try { + expect(await cursor.getAll()).toEqual([{ name: 'CodeEmbedding' }]); + } finally { + cursor.close(); + } } finally { - cursor.close(); + await closeLbugConnection(exported, { strict: true, databasePath: receipt.path }); } - } finally { - await closeLbugConnection(exported, { strict: true, databasePath: receipt.path }); - } - await fs.mkdir(path.join(f.root, 'reuse')); - store = await VectorStore.open({ - directory: path.join(f.root, 'reuse'), - identity: 'b'.repeat(64), - dimensions: 4, - mode: 'create', - bufferPoolBytes: pool, - threads: 1, - batchBytes: 65536, - }); - source = await LegacyVectorSource.open( - { - path: receipt.path, - sourceIdentity: receipt.identity, - recordedIdentity: receipt.recordedIdentity, - expectedIdentity: identity, - expectedInputIdentity: 'c'.repeat(64), - config, + await fs.mkdir(path.join(f.root, 'reuse')); + store = await VectorStore.open({ + directory: path.join(f.root, 'reuse'), + identity: 'b'.repeat(64), + dimensions: 4, + mode: 'create', bufferPoolBytes: pool, threads: 1, batchBytes: 65536, - }, - store, - receipt.excludedIds, - ); - const group = await source.reusable(node.id, f.hash, { node }); - expect(group?.chunks).toBe(1); - expect(await source.readChunks(group!, 0, 1)).toEqual([ - { chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0, 0, 0] }, - ]); - } finally { - await source?.close(); - await store?.close(); - await f.close(); - } -}); + }); + source = await LegacyVectorSource.open( + { + path: receipt.path, + sourceIdentity: receipt.identity, + recordedIdentity: receipt.recordedIdentity, + expectedIdentity: identity, + expectedInputIdentity: 'c'.repeat(64), + config, + bufferPoolBytes: pool, + threads: 1, + batchBytes: 65536, + }, + store, + receipt.excludedIds, + ); + const group = await source.reusable(node.id, f.hash, { node }); + expect(group?.chunks).toBe(1); + expect(await source.readChunks(group!, 0, 1)).toEqual([ + { chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0, 0, 0] }, + ]); + } finally { + await source?.close(); + await store?.close(); + await f.close(); + } + }, +); it('keeps vector payloads out of JavaScript while transferring a larger table', async () => { const f = await fixture(4096); From 25e5a869006ffaa4f18f3acbc3cdf3eb9f468708 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 18:51:15 +0000 Subject: [PATCH 57/67] test: restore watch resource mock contract Provide the supervisor reservation used by watch refresh admission. Refs #3829. --- gitnexus/test/unit/watch-paths.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/gitnexus/test/unit/watch-paths.test.ts b/gitnexus/test/unit/watch-paths.test.ts index 3e39deea6..4effbfdd1 100644 --- a/gitnexus/test/unit/watch-paths.test.ts +++ b/gitnexus/test/unit/watch-paths.test.ts @@ -12,6 +12,7 @@ import { vi.mock('../../src/core/analysis-host-resources.js', () => ({ analysisMayEmbed: vi.fn(async () => false), + analysisSupervisorReservation: () => 192 * 1024 ** 2, resolveHostAnalysisResources: vi.fn(() => ({ mainHeapMb: 512 })), })); From 2b96ccdbd87cb78da7994c782730bff7515d826a Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 19:04:00 +0000 Subject: [PATCH 58/67] fix: retain watch budgets and bound lookahead Keep explicit heap reservations across refreshes and limit brace scans to the existing three-line window. Refs #3829. --- gitnexus/src/cli/analyze-watch.ts | 4 +++- .../src/core/embeddings/text-generator.ts | 3 +-- .../unit/watch-supervisor-reservation.test.ts | 19 ++++++++++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index 30352ac4a..7a56ba7d1 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -196,7 +196,9 @@ export async function resolveWatchOptions( ); const resources = resolveHostAnalysisResources({ totalMb: positiveInteger(merged.resourceBudget, '--resource-budget'), - mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + mainHeapMb: + positiveInteger(cli.memoryBudget, '--memory-budget') ?? + Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), workers: workerPoolSize, includeEmbeddings: await analysisMayEmbed(repoPath, { branch }), supervisorBytes: diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index 32bf70a3c..b9b6c396d 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -319,8 +319,7 @@ export const extractDeclarationOnly = (content: string): string => { } lookaheadEnd = end + 1; } - const nextBrace = content.indexOf('{', nextLineStart); - if (!trimmed.includes('{') && !(nextBrace >= 0 && nextBrace < lookaheadEnd)) { + if (!trimmed.includes('{') && !content.slice(nextLineStart, lookaheadEnd).includes('{')) { return ''; } started = true; diff --git a/gitnexus/test/unit/watch-supervisor-reservation.test.ts b/gitnexus/test/unit/watch-supervisor-reservation.test.ts index 3f7589bae..557b2263a 100644 --- a/gitnexus/test/unit/watch-supervisor-reservation.test.ts +++ b/gitnexus/test/unit/watch-supervisor-reservation.test.ts @@ -64,20 +64,29 @@ it.each([undefined, String(320 * 1024 ** 2)])( }, ); -it.each([undefined, String(320 * 1024 ** 2)])( - 'retains supervisor reservation %s across watch refreshes', - async (inherited) => { +it.each([ + [undefined, undefined], + [String(320 * 1024 ** 2), undefined], + [undefined, '768'], + [String(320 * 1024 ** 2), '768'], +])( + 'retains supervisor %s and explicit heap %s across watch refreshes', + async (inherited, memoryBudget) => { const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); directories.push(directory); vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', inherited); vi.stubEnv('GITNEXUS_RC', ''); const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; - await resolveWatchOptions(directory, {}, baseline); - await resolveWatchOptions(directory, {}, baseline); + await resolveWatchOptions(directory, { memoryBudget }, baseline); + await resolveWatchOptions(directory, { memoryBudget }, baseline); expect(mocks.resources).toHaveBeenCalledTimes(2); for (const [request] of mocks.resources.mock.calls) expect(request).toMatchObject({ supervisorBytes: inherited === undefined ? 192 * 1024 ** 2 : Number(inherited), + mainHeapMb: + memoryBudget === undefined + ? Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2) + : Number(memoryBudget), }); }, ); From 6492175afda1719e2768d29aaaf282f45c7569ef Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 19:31:41 +0000 Subject: [PATCH 59/67] fix: stop retries on shutdown and allow empty parse --- .../ingestion/pipeline-phases/parse-impl.ts | 1 + gitnexus/src/server/analyze-job.ts | 6 +++++- .../unit/analyze-launch-branch-settle.test.ts | 21 +++++++++++++++++++ .../unit/parse-impl-worker-lazy-cache.test.ts | 18 ++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 818d08aa8..9d5879cf7 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -618,6 +618,7 @@ export async function runChunkedParseAndResolve( // spawns the full requested pool. const workerResources = options?.workerResources; if ( + totalParseable > 0 && workerResources && (!Number.isSafeInteger(workerResources.workers) || workerResources.workers < 1 || diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index 73fd1c3aa..555f6101f 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -361,9 +361,13 @@ export class JobManager { } dispose() { + // Cancel jobs between workers too, so an already scheduled retry cannot fork. + for (const job of this.jobs.values()) { + if (!this.hasPendingCancel(job.id)) this.cancelJob(job.id, 'Server shutting down'); + } // Cooperative cancellation precedes bounded standalone-process shutdown. for (const [jobId, child] of this.children) { - this.requestChildShutdown(jobId, child); + if (!this.hasPendingCancel(jobId)) this.requestChildShutdown(jobId, child); } for (const controller of this.abortControllers.values()) controller.abort(); this.abortControllers.clear(); diff --git a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts index 1e062de63..5bf25705f 100644 --- a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts +++ b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts @@ -437,6 +437,27 @@ describe('finalization gate follows the placement the run chose', () => { expect(order).toEqual(['backend.init', 'releaseRepoLock']); }); + it.each(['before-exit', 'during-retry'] as const)( + 'does not restart a worker when shutdown begins %s', + async (when) => { + vi.useFakeTimers(); + const releaseRepoLock = vi.fn(); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher({ releaseRepoLock })(job, REPO_PATH, {}); + + if (when === 'before-exit') jobManager.dispose(); + child.emit('exit', 1); + if (when === 'during-retry') jobManager.dispose(); + await jobManager.waitForChildren(); + await vi.advanceTimersByTimeAsync(5_000); + + expect(forkMock).toHaveBeenCalledTimes(1); + expect(jobManager.getJob(job.id)?.status).toBe('failed'); + expect(jobManager.getJob(job.id)?.error).toBe('Server shutting down'); + expect(releaseRepoLock).toHaveBeenCalledTimes(1); + }, + ); + it('still treats an exit with no terminal IPC as a crash worth retrying', async () => { // The guard must not swallow real crashes: no `complete`/`error` was sent. H.settledDir = H.STORAGE_PATH; diff --git a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts index ce7423d6d..fa5d661af 100644 --- a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts +++ b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts @@ -341,6 +341,24 @@ describe('parse-impl worker pool lazy startup', () => { } }); + it('accepts a zero-worker budget when there are no parseable files', async () => { + const result = await runChunkedParseAndResolve( + createKnowledgeGraph(), + [], + [], + 0, + repoDir, + Date.now(), + () => {}, + { + workerPoolSize: 0, + workerResources: { workers: 0, workerHeapMb: 512 }, + workerUrlForTest: pathToFileURL(path.join(tempDir, 'must-not-spawn.cjs')), + }, + ); + expect(result.usedWorkerPool).toBe(false); + }); + it('keeps tiny-repository automatic sizing inside the admitted worker ceiling', async () => { const saved = process.env.GITNEXUS_WORKER_POOL_SIZE; delete process.env.GITNEXUS_WORKER_POOL_SIZE; From c2f7e03c38b66cd9abc13877dbd8eedcacdf6b72 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 19:57:40 +0000 Subject: [PATCH 60/67] refactor: simplify embedding setup and test lists --- gitnexus/src/core/embeddings/ast-utils.ts | 1 - gitnexus/src/core/embeddings/embedding-job.ts | 3 +-- .../src/core/embeddings/embedding-pipeline.ts | 1 - .../src/core/embeddings/text-generator.ts | 20 +++++++++---------- gitnexus/vitest.config.ts | 12 +++-------- 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts index 2f4bac24e..11315ef7d 100644 --- a/gitnexus/src/core/embeddings/ast-utils.ts +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -19,7 +19,6 @@ const parserCache = new Map(); * Returns null if language is unavailable or parsing fails. */ export const ensureAndParse = async (content: string, filePath: string): Promise => { - // Fixed chunks need no provider registry. Load it only when parsing a snippet. const { getLanguageForFileContent, getProvider } = await import('../ingestion/languages/index.js'); // Same classifier as ingest. Filename-only maps `.h` → C++, so Objective-C diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts index ca2eb871e..85a5e34a4 100644 --- a/gitnexus/src/core/embeddings/embedding-job.ts +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -12,7 +12,6 @@ import { } from '../../storage/embedding-job.js'; import { adoptEmbeddingArtifact, - embeddingArtifactSpaceKey, readEmbeddingArtifactSpace, type EmbeddingArtifactOwner, } from '../../storage/embedding-artifact.js'; @@ -429,7 +428,7 @@ export async function runPublishedEmbeddingJob( identity: identities.identity, graphGeneration: graph.id, artifactIdentity: owner.artifactIdentity, - spaceKey: embeddingArtifactSpaceKey(space), + spaceKey: artifact.spaceKey, }, }; await options.metadata.saveMetadata(metadata); diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 1f2a64e68..a254c20c3 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -116,7 +116,6 @@ export const contentHashForNode = ( ): string => { // Hash must be deterministic across runs, so exclude methodNames/fieldNames // which are populated during the batch loop via AST extraction. - // Using only node.content ensures the hash stays stable. // NOTE: A change to extractStructuralNames behavior requires bumping EMBEDDING_TEXT_VERSION. const hashNode = { ...node, methodNames: undefined, fieldNames: undefined }; if (node.label === 'Class') hashNode.classDeclaration = undefined; diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index b9b6c396d..8af6ee5de 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -310,17 +310,17 @@ export const extractDeclarationOnly = (content: string): string => { if (!started) { if (DECL_START_RE.test(trimmed)) { // Non-brace language check: current line or next 3 lines must have `{` - let lookaheadEnd = nextLineStart; - for (let i = 0; i < 3; i++) { - const end = content.indexOf('\n', lookaheadEnd); - if (end < 0) { - lookaheadEnd = content.length; - break; + if (!trimmed.includes('{')) { + let lookaheadEnd = nextLineStart; + for (let i = 0; i < 3; i++) { + const end = content.indexOf('\n', lookaheadEnd); + if (end < 0) { + lookaheadEnd = content.length; + break; + } + lookaheadEnd = end + 1; } - lookaheadEnd = end + 1; - } - if (!trimmed.includes('{') && !content.slice(nextLineStart, lookaheadEnd).includes('{')) { - return ''; + if (!content.slice(nextLineStart, lookaheadEnd).includes('{')) return ''; } started = true; declLines.append(trimmed); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 45f3edac1..624e552a8 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -2,10 +2,10 @@ import { defineConfig } from 'vitest/config'; import PerfSequencer from './test/helpers/perf-sequencer.js'; // Keep native tests in exactly one project (TESTING.md). -const EMBEDDING_VECTOR_STORE_TEST = 'test/integration/embedding-vector-store.test.ts'; -const LEGACY_VECTOR_REUSE_TEST = 'test/integration/legacy-vector-reuse-index.test.ts'; -const LEGACY_VECTOR_EXPORT_TEST = 'test/integration/legacy-vector-export.test.ts'; const RESUMABLE_ANALYSIS_NATIVE_TESTS = [ + 'test/integration/embedding-vector-store.test.ts', + 'test/integration/legacy-vector-reuse-index.test.ts', + 'test/integration/legacy-vector-export.test.ts', 'test/integration/analysis-embedding-flow.test.ts', 'test/integration/analysis-generation-routing.test.ts', 'test/integration/analysis-publication-flow.test.ts', @@ -93,9 +93,6 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ - EMBEDDING_VECTOR_STORE_TEST, - LEGACY_VECTOR_REUSE_TEST, - LEGACY_VECTOR_EXPORT_TEST, ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', @@ -215,9 +212,6 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ - EMBEDDING_VECTOR_STORE_TEST, - LEGACY_VECTOR_REUSE_TEST, - LEGACY_VECTOR_EXPORT_TEST, ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', From 2ca954fd35961a44d56bf7fe44369956087becc6 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 06:11:51 +0000 Subject: [PATCH 61/67] perf: batch vector completion hash updates Keep hash finalization inside the existing transaction and row bounds. Run embedding input-isolation tests in the serial native project. --- gitnexus/src/core/lbug/vector-store.ts | 36 +++++++++++-------- .../embedding-vector-store.test.ts | 28 +++++++++++---- gitnexus/vitest.config.ts | 1 + 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index 28dceb62f..c8074c259 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -636,29 +636,35 @@ export class VectorStore { } await this.verify(verified); const finalize = async () => { + let ids: string[] = []; + let hashes: string[] = []; + const flush = async () => { + if (!ids.length) return; + await this.query( + 'UNWIND range(1, size($ids)) AS i MATCH (e:CodeEmbedding {id:$ids[i]}) SET e.contentHash=$hashes[i]', + { ids, hashes }, + ); + ids = []; + hashes = []; + }; for (let i = 0; i < nodes.length; i++) { const node = nodes[i]; if (!node.inputsDigest) continue; const contentHash = `${node.contentHash}:${node.inputsDigest}`; - for (let start = 0; start < node.chunks; start += BLOCK_ROWS) { - const ids = Array.from( - { length: Math.min(BLOCK_ROWS, node.chunks - start) }, - (_, j) => `${node.nodeId}:${start + j}`, - ); - await this.query( - 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.contentHash=$contentHash', - { ids, contentHash }, - ); + for (let ordinal = 0; ordinal < node.chunks; ordinal++) { + ids.push(`${node.nodeId}:${ordinal}`); + hashes.push(contentHash); + if (ids.length === BLOCK_ROWS) await flush(); } - await this.query( - 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash', - { nodeId: node.nodeId, contentHash }, - ); verified[i] = { ...verified[i], contentHash }; } + await flush(); await this.query( - 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', - { ids: nodes.map((n) => n.nodeId) }, + 'UNWIND range(1, size($ids)) AS i MATCH (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.complete=true', + { + ids: verified.map((receipt) => receipt.nodeId), + hashes: verified.map((receipt) => receipt.contentHash), + }, ); }; if (nodes.some((node) => node.inputsDigest)) await this.transaction(finalize); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts index d052dcf36..26f58ef5f 100644 --- a/gitnexus/test/integration/embedding-vector-store.test.ts +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -920,24 +920,38 @@ describe('separate durable vector artifacts', () => { expect(await store.reusable(node.nodeId, node.contentHash)).toBeNull(); }); - it('writes a multi-node unit and verifies a large node in bounded batches', async () => { + it('finalizes distinct input hashes across chunk batches and preserves unhashed groups on reopen', async () => { const options = await setup(); - const nodes = [0, 1].map((i) => ({ nodeId: `Function:a:node${i}`, contentHash: 'hash' })); + const nodes = [0, 1, 2].map((i) => ({ + nodeId: `Function:a:node${i}`, + contentHash: String(i).repeat(40), + })); await store.beginNodes(nodes); - for (let start = 0; start < 600; start += 128) { - const rows = Array.from({ length: Math.min(128, 600 - start) }, (_, i) => ({ - ...nodes[start + i < 300 ? 0 : 1], + for (let start = 0; start < 601; start += 128) { + const rows = Array.from({ length: Math.min(128, 601 - start) }, (_, i) => ({ + ...nodes[Math.floor((start + i) / 300)], ...chunk((start + i) % 300), })); await store.appendBatch(rows); } - const receipts = await store.finishNodes(nodes.map((node) => ({ ...node, chunks: 300 }))); + const receipts = await store.finishNodes( + nodes.map((node, i) => ({ + ...node, + chunks: i < 2 ? 300 : 1, + inputsDigest: i < 2 ? String(i + 3).repeat(40) : undefined, + })), + ); + expect(receipts.map((receipt) => receipt.contentHash)).toEqual([ + `${nodes[0].contentHash}:${'3'.repeat(40)}`, + `${nodes[1].contentHash}:${'4'.repeat(40)}`, + nodes[2].contentHash, + ]); await store.checkpointUnit(receipts); await store.close(); store = await VectorStore.open({ ...options, mode: 'resume' }); for (const receipt of receipts) expect(await store.reusable(receipt.nodeId, receipt.contentHash)).toEqual(receipt); - expect(await store.seal(2, 600)).toBe('exact'); + expect(await store.seal(3, 601)).toBe('exact'); }); it('publishes an empty artifact without loading a model or ANN extension', async () => { diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 624e552a8..e29826259 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -3,6 +3,7 @@ import PerfSequencer from './test/helpers/perf-sequencer.js'; // Keep native tests in exactly one project (TESTING.md). const RESUMABLE_ANALYSIS_NATIVE_TESTS = [ + 'test/integration/embedding-input-isolation.test.ts', 'test/integration/embedding-vector-store.test.ts', 'test/integration/legacy-vector-reuse-index.test.ts', 'test/integration/legacy-vector-export.test.ts', From 6e01aa1a715120d5566c849e5906148e39b1f24f Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 06:27:42 +0000 Subject: [PATCH 62/67] fix: address PR #3829 recovery feedback Preserve terminal worker results, reclaim orphan attempt metadata, retain artifacts with unrecognized space ownership, and correct the cancellation comment. --- .../core/auto-sync/analysis-worker-launch.ts | 1 + gitnexus/src/server/analyze-job.ts | 6 +- .../src/storage/analysis-artifact-cleanup.ts | 5 ++ gitnexus/src/storage/embedding-artifact.ts | 3 +- .../test/unit/analysis-publication.test.ts | 64 +++++++++++++++++++ .../unit/auto-sync-analysis-worker.test.ts | 35 ++++++++++ 6 files changed, 109 insertions(+), 5 deletions(-) diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 57ec1d2ec..a2a78ad66 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -259,6 +259,7 @@ export function createAutoSyncAnalysisRunner( return; } terminalOutcome = message; + deps.clearTimeoutFn(timeout); }); child.on('error', (error) => { const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index 555f6101f..f13eb9150 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -294,10 +294,8 @@ export class JobManager { /** * Cancel a running job. * - * The worker is asked to stop over IPC first (`{ type: 'cancel' }`), which - * lets it reach a JS-visible safe point and checkpoint before exiting — - * the same cross-platform control path `core/auto-sync` uses. Native work - * must reach a safe point even if that takes longer than the grace period. + * Request cooperative cancellation over IPC first. A standalone child that + * outlasts the grace period is killed; ownership remains held until its exit. */ cancelJob(jobId: string, reason?: string): boolean { const job = this.jobs.get(jobId); diff --git a/gitnexus/src/storage/analysis-artifact-cleanup.ts b/gitnexus/src/storage/analysis-artifact-cleanup.ts index 5681f257a..7d121dbfc 100644 --- a/gitnexus/src/storage/analysis-artifact-cleanup.ts +++ b/gitnexus/src/storage/analysis-artifact-cleanup.ts @@ -8,6 +8,7 @@ import { readLegacyEmbeddingSource } from './legacy-embedding-source.js'; import { logger } from '../core/logger.js'; const GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/; +const ATTEMPT_METADATA = /^analysis-meta-[a-f0-9]{32}\.json$/; const ID = /^[a-f0-9]{64}$/; /** Keep the published and previous graph, all model spaces, pending jobs, and @@ -96,6 +97,10 @@ export async function reclaimAnalysisArtifacts( return false; } }; + // Without a publication receipt, no attempt metadata is needed for recovery. + for (const entry of entries) + if (entry.isFile() && ATTEMPT_METADATA.test(entry.name)) + await remove(path.join(root, entry.name)); // A busy sidecar may outlive its base graph; include it in the next sweep. const candidates = new Set( entries diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts index 47f30c25c..569aaaf83 100644 --- a/gitnexus/src/storage/embedding-artifact.ts +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -53,7 +53,8 @@ export async function readEmbeddingArtifactOwners(root: string): Promise { const raw = value as EmbeddingArtifactRecord; diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts index 1f085565a..31ad737a0 100644 --- a/gitnexus/test/unit/analysis-publication.test.ts +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -99,6 +99,70 @@ describe('durable graph publication', () => { expect(graphs).toContain((await loadMeta(root))?.graphPath); }); + it('reclaims attempt metadata orphaned before its publication receipt was written', async () => { + const { root, stage } = await fixture(); + const rename = fs.rename.bind(fs); + const interrupted = vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + if (path.basename(String(target)) === 'analysis-publication.json') + throw new Error('Interrupted before receipt'); + await rename(source, target); + }); + await expect(prepareAnalysisPublication(root, stage, metadata, content)).rejects.toThrow( + 'Interrupted before receipt', + ); + interrupted.mockRestore(); + const orphan = (await fs.readdir(root)).find((name) => + /^analysis-meta-[a-f0-9]{32}\.json$/.test(name), + ); + expect(orphan).toBeDefined(); + await fs.writeFile(path.join(root, 'analysis-meta-not-owned.json'), 'keep'); + await reclaimAnalysisArtifacts(root, metadata); + expect(await fs.readdir(root)).not.toContain(orphan); + expect(await fs.readFile(path.join(root, 'analysis-meta-not-owned.json'), 'utf8')).toBe('keep'); + expect(await loadMeta(root)).toEqual(metadata); + + const pending = await prepareAnalysisPublication(root, stage, metadata, content); + await reclaimAnalysisArtifacts(root, metadata); + expect(await readPreparedAnalysis(root)).toEqual(pending); + expect(await fs.readFile(path.join(root, pending.metadataFile), 'utf8')).toContain(content); + }); + + it('retains paid vectors when an ownership space was renamed', async () => { + const { root } = await fixture(); + const identity = 'a'.repeat(64); + const job = path.join(root, 'embeddings', identity); + await fs.mkdir(job, { recursive: true }); + await writeEmbeddingJobRecord(job, 'job.json', { version: 1, identity }); + await fs.writeFile(path.join(job, 'vectors'), 'paid vectors'); + const record = await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: identity, + jobIdentity: identity, + graphGeneration: 'd'.repeat(32), + sourceIdentity: 'e'.repeat(64), + manifestDigest: 'f'.repeat(64), + phase: 'pending', + revalidateInputs: false, + }, + async () => {}, + ); + const parent = path.join(root, 'embeddings', 'spaces'); + await fs.rename(path.join(parent, record.spaceKey), path.join(parent, 'renamed-space')); + const warn = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, warn); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Unexpected embedding space entry')); + expect(await fs.readFile(path.join(job, 'vectors'), 'utf8')).toBe('paid vectors'); + expect(await fs.readFile(path.join(job, 'job.json'), 'utf8')).toContain(identity); + }); + it.each(['pending', 'sealed'] as const)( 'reclaims unreferenced jobs and obsolete graphs for a %s model owner', async (phase) => { diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 589785090..3575ca28b 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -341,6 +341,41 @@ describe('auto-sync analysis worker', () => { expect(timers).toHaveLength(1); }); + it.each(['complete', 'error'] as const)( + 'preserves a terminal %s received before the deadline while waiting for exit', + async (type) => { + vi.useFakeTimers(); + try { + const child = createChild(); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: setTimeout, + clearTimeoutFn: clearTimeout, + }); + const result = run('/tmp/repo', {}, 50).catch((error: unknown) => error); + let settled = false; + void result.then(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(49); + child.emit( + 'message', + type === 'complete' + ? { type, result: { stats: { files: 3 } } } + : { type, message: 'worker failed' }, + ); + await vi.advanceTimersByTimeAsync(500); + expect(settled).toBe(false); + expect(child.send).toHaveBeenCalledTimes(1); + child.emit('exit', 0, null); + if (type === 'complete') await expect(result).resolves.toEqual({ stats: { files: 3 } }); + else await expect(result).resolves.toMatchObject({ message: 'worker failed' }); + } finally { + vi.useRealTimers(); + } + }, + ); + it('divides the worker heap by the number of repos analyzed in parallel', async () => { // Stubbed timers so neither run leaves a live timeout behind for the rest // of the suite, and both promises are settled before the test returns. From 8482d42d43f48f552a436de160d27bb0689905fa Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 06:30:54 +0000 Subject: [PATCH 63/67] perf: retain simple vector completion query --- gitnexus/src/core/lbug/vector-store.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index c8074c259..d538c3676 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -668,7 +668,11 @@ export class VectorStore { ); }; if (nodes.some((node) => node.inputsDigest)) await this.transaction(finalize); - else await finalize(); + else + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', + { ids: nodes.map((node) => node.nodeId) }, + ); return verified.map((receipt) => ({ ...receipt, complete: true })); }, true); } From 07dfbc979e878ac6b690b675fa44eb4f35e37dc5 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 06:42:17 +0000 Subject: [PATCH 64/67] perf: skip redundant alias identity probes --- gitnexus/src/core/run-analyze.ts | 4 ++-- gitnexus/src/storage/repo-manager.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 471f3dd33..3d7a0de79 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1482,8 +1482,8 @@ async function preparedAnalysisName(repoPath: string, options: AnalyzeOptions): if ( existing && existing.name !== path.basename(path.resolve(existing.path)) && - existing.name !== path.basename(resolveRepoIdentityRoot(existing.path)) && - existing.name !== inferred + existing.name !== inferred && + existing.name !== path.basename(resolveRepoIdentityRoot(existing.path)) ) return existing.name; return inferred ?? path.basename(resolveRepoIdentityRoot(repoPath)); diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index d338b71ae..96ea04498 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -944,6 +944,7 @@ export class RegistryNameCollisionError extends Error { const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): boolean => { const resolved = path.resolve(entry.path); if (entry.name === path.basename(resolved)) return false; + if (inferredName && entry.name === inferredName) return false; // Canonical-root-derived names are not user aliases either (#1259): // a worktree registered under the canonical repo's basename // (e.g. `{name: 'repo', path: '/repo/wt-feature'}`) must re-register @@ -955,7 +956,6 @@ const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): bool // — analyze canonical, then analyze worktree, then re-analyze worktree // — would break on the third call. if (entry.name === path.basename(resolveRepoIdentityRoot(resolved))) return false; - if (inferredName && entry.name === inferredName) return false; return true; }; From 50ffdd7be796cd719d8a2a290fba00e017b5d5bb Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 06:51:54 +0000 Subject: [PATCH 65/67] perf: avoid Git probes for ordinary checkout roots --- gitnexus/src/storage/git.ts | 17 +++++++++++- gitnexus/test/unit/git-utils.test.ts | 39 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 72f59f618..c3c9317bd 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -1,5 +1,5 @@ import { execFileSync, execSync, spawnSync } from 'child_process'; -import { statSync, existsSync } from 'fs'; +import { statSync, existsSync, lstatSync, realpathSync } from 'fs'; import path from 'path'; import os from 'os'; import { logger } from '../core/logger.js'; @@ -585,6 +585,21 @@ export const getCoreExcludesFilePath = (fromPath: string): string => { */ export const resolveRepoIdentityRoot = (fromPath: string): string => { const resolved = path.resolve(fromPath); + // An ordinary checkout is its own identity root. Let Git interpret worktrees, + // symlinks, common-directory indirection and environment overrides as before. + try { + const gitDir = path.join(resolved, '.git'); + if ( + process.env.GIT_DIR === undefined && + process.env.GIT_COMMON_DIR === undefined && + lstatSync(gitDir).isDirectory() && + !existsSync(path.join(gitDir, 'commondir')) && + realpathSync.native(resolved) === resolved + ) + return resolved; + } catch { + // Missing or unreadable paths retain the existing Git/fallback behavior. + } const canonical = getCanonicalRepoRoot(resolved); if (!canonical) return resolved; // non-git → use as-is if (canonical === resolved) return canonical; // canonical checkout diff --git a/gitnexus/test/unit/git-utils.test.ts b/gitnexus/test/unit/git-utils.test.ts index b1e00d727..4ea785b4d 100644 --- a/gitnexus/test/unit/git-utils.test.ts +++ b/gitnexus/test/unit/git-utils.test.ts @@ -416,6 +416,45 @@ describe('getCanonicalRepoRoot', () => { }); }); +describe('resolveRepoIdentityRoot', () => { + it.each(['ordinary', 'GIT_DIR', 'GIT_COMMON_DIR', 'commondir', 'symlink'] as const)( + 'preserves Git identity for %s checkouts', + async (layout) => { + const { getCanonicalRepoRoot, resolveRepoIdentityRoot } = + await import('../../src/storage/git.js'); + const root = fs.realpathSync.native(makeIsolatedTempDir('gitnexus-identity-')); + let repo = path.join(root, 'repo'); + const other = path.join(root, 'other'); + vi.stubEnv('GIT_DIR', undefined); + vi.stubEnv('GIT_COMMON_DIR', undefined); + try { + for (const directory of [repo, other]) { + fs.mkdirSync(directory); + execFileSync(gitExecutable, ['init', '-q'], { cwd: directory, stdio: 'pipe' }); + } + if (layout === 'GIT_DIR' || layout === 'GIT_COMMON_DIR') + vi.stubEnv(layout, path.join(other, '.git')); + else if (layout === 'commondir') + fs.writeFileSync(path.join(repo, '.git', 'commondir'), path.join(other, '.git') + '\n'); + else if (layout === 'symlink') { + const link = path.join(root, 'linked-name'); + fs.symlinkSync(repo, link, 'junction'); + repo = link; + } + const canonical = getCanonicalRepoRoot(repo); + expect(canonical).not.toBeNull(); + expect(resolveRepoIdentityRoot(repo)).toBe(canonical); + const subdir = path.join(repo, 'subdir'); + fs.mkdirSync(subdir); + expect(resolveRepoIdentityRoot(subdir)).toBe(path.resolve(subdir)); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); + // ─── selfCommitContextFiles (#2639) ──────────────────────────────────────── describe('selfCommitContextFiles', () => { From 68508becf2d7129566d95789f263f453f8413e03 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 07:02:16 +0000 Subject: [PATCH 66/67] perf: avoid redundant vector transactions Standalone writes already execute in automatic transactions. Keep multi-query transactions and durable unit checkpoints unchanged. --- gitnexus/src/core/lbug/vector-store.ts | 34 +++++++++++--------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts index d538c3676..85a9f9cfa 100644 --- a/gitnexus/src/core/lbug/vector-store.ts +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -378,12 +378,10 @@ export class VectorStore { ) throw new Error('Cannot select an incomplete vector group'); await this.ensureSelectionTable(); - await this.transaction(async () => { - await this.query( - 'UNWIND $ids AS id MERGE (s:EmbeddingSelection {nodeId:id}) SET s.job=$job', - { ids: [...ids], job }, - ); - }); + await this.query( + 'UNWIND $ids AS id MERGE (s:EmbeddingSelection {nodeId:id}) SET s.job=$job', + { ids: [...ids], job }, + ); }, true); } @@ -466,9 +464,7 @@ export class VectorStore { let ids: string[] = []; const remove = async () => { if (!ids.length) return; - await this.transaction(async () => { - await this.query('UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) DELETE e', { ids }); - }); + await this.query('UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) DELETE e', { ids }); ids = []; }; for (const receipt of previous.values()) @@ -478,17 +474,15 @@ export class VectorStore { } await remove(); } - await this.transaction(async () => { - await this.query( - 'UNWIND range(1, size($ids)) AS i MERGE (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.chunks=0, n.complete=false, n.sourceStartLine=$firstLines[i], n.sourceEndLine=$lastLines[i]', - { - ids: nodes.map((node) => node.nodeId), - hashes: nodes.map((node) => node.contentHash), - firstLines: nodes.map((node) => node.sourceStartLine ?? 0), - lastLines: nodes.map((node) => node.sourceEndLine ?? 0), - }, - ); - }); + await this.query( + 'UNWIND range(1, size($ids)) AS i MERGE (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.chunks=0, n.complete=false, n.sourceStartLine=$firstLines[i], n.sourceEndLine=$lastLines[i]', + { + ids: nodes.map((node) => node.nodeId), + hashes: nodes.map((node) => node.contentHash), + firstLines: nodes.map((node) => node.sourceStartLine ?? 0), + lastLines: nodes.map((node) => node.sourceEndLine ?? 0), + }, + ); }, true); } From aa3e90e89e780451d2ea1b8d7a8ab45f6995558f Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 7 Nov 2026 07:14:10 +0000 Subject: [PATCH 67/67] docs: clarify cancellation grace period --- gitnexus/src/server/analyze-job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index f13eb9150..35b073e22 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -89,7 +89,7 @@ export interface AnalyzeJob { const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const JOB_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes -/** How long to retain event-loop references after requesting safe cancellation. */ +/** Delay before force-killing an unresponsive standalone child after cancellation. */ const CANCEL_GRACE_MS = 60_000; export class JobManager {