diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c0aa0fe3d..b5ef48529 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -7,6 +7,95 @@ permissions: contents: read jobs: + analyze-resume-benchmark: + name: analyze / resumable performance (${{ matrix.resources }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + resources: [explicit, default] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: Pin comparison revision + id: baseline + shell: bash + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [[ -n "$PR_BASE_SHA" ]]; then + comparison_sha="$PR_BASE_SHA" + else + comparison_sha="$(git rev-parse --verify HEAD^1)" + fi + printf 'sha=%s\n' "$comparison_sha" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/setup-gitnexus + with: + build: 'true' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.baseline.outputs.sha }} + path: baseline + persist-credentials: false + - name: Build baseline runtime + working-directory: baseline/gitnexus + run: | + npm ci + npm run build + npx tsx scripts/ensure-fts.ts + - name: Install candidate search extensions + working-directory: gitnexus + run: npx tsx scripts/ensure-fts.ts + - name: Profile fresh HTTP analysis separately from acceptance timings + if: matrix.resources == 'explicit' + working-directory: gitnexus + run: | + mkdir -p "$RUNNER_TEMP/analyze-profile" + node bench/analyze-resume/measure.mjs --cpu-profile --allow-dependency-changes \ + --scenarios http-fresh --pairs 1 --warmups 0 \ + --baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \ + --output "$RUNNER_TEMP/analyze-profile/results.json" + - name: Retain diagnostic CPU profiles before the full comparison + if: always() && matrix.resources == 'explicit' + uses: ./.github/actions/upload-test-artifact + with: + name: analyze-resume-cpu-profile-${{ matrix.resources }} + path: | + ${{ runner.temp }}/analyze-profile/**/*.cpuprofile + ${{ runner.temp }}/analyze-profile/**/*.json + ${{ runner.temp }}/analyze-profile/**/*.log + retention-days: 7 + - name: Check paired analyze and resume performance + working-directory: gitnexus + shell: bash + env: + RESOURCE_PROFILE: ${{ matrix.resources }} + run: | + mkdir -p "$RUNNER_TEMP/analyze-resume" + resource_args=() + if [[ "$RESOURCE_PROFILE" == default ]]; then + resource_args+=(--default-resources) + fi + node bench/analyze-resume/measure.mjs --validate-gate + node bench/analyze-resume/measure.mjs --check --allow-dependency-changes \ + --baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \ + --output "$RUNNER_TEMP/analyze-resume/results.json" \ + "${resource_args[@]}" 2>&1 | tee "$RUNNER_TEMP/analyze-resume/run.log" + - name: Retain paired measurements and analyzer logs + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: analyze-resume-performance-${{ matrix.resources }} + path: | + ${{ runner.temp }}/analyze-resume/*.json + ${{ runner.temp }}/analyze-resume/*.log + ${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.json + ${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.log + retention-days: 30 + lexical-lookup-benchmark: name: indexing / lexical lookup and cache interning runs-on: ubuntu-latest diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0d9e32b18..8c4d30b67 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -640,25 +640,31 @@ Inheritance is captured by the `@reference.inherits` tag and emitted by the scop ``` CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) - 1. Early exit if lastCommit == HEAD (unless --force) [0%] - 2. Cache existing embeddings from prior index [0%] - 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] - 4. Clean up legacy KuzuDB files [60%] - 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] - 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] - 7. Restore cached embeddings (batch insert) [88%] - 8. Generate new embeddings if --embeddings [90-98%] - 9. Save metadata + register repo + update .gitignore [98-100%] - 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] + 1. Admit phase resources and recover any prepared graph publication. + 2. Reuse unchanged graph/parser work where its receipts still match. + 3. runPipelineFromRepo() → KnowledgeGraph → bounded graph write → FTS. + 4. Prepare the owned graph and publication receipt; drain native resources. + 5. Supervisor waits for graph-process exit, verifies and publishes metadata. + 6. If embeddings are requested/pending, open the resumable embedding job. + 7. Reuse verified vectors, checkpoint bounded units, then seal/publish vectors. + 8. Register the repo and generate AI context files. + +Graph-only legacy slots keep their existing path until separate publication is +needed. A ready graph remains queryable while an embedding job is pending; +semantic search reports degradation until a compatible sealed artifact is ready. ``` -**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. +**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, resumable), `--skipGit`, `--noStats`. ## Storage ``` /.gitnexus/ - ├── lbug # LadybugDB database + ├── lbug # Legacy monolithic database, until first separate publication + ├── lbug.generation. # Owned private graph, selected by graphPath in metadata + ├── analysis-publication.json # Recoverable prepared graph receipt + ├── embeddings// # Job journal, manifest and separate vector database + ├── embeddings/spaces/ # Model-space ownership/reuse receipts ├── lbug.wal # Write-ahead log ├── lbug.shadow # Shadow sidecar (checkpoint staging) ├── lbug.lock # Single-writer lock @@ -690,7 +696,11 @@ the read-only open is retried — see `sidecar-recovery.ts` (`isReadOnlyCheckpointInProgressError`) and the `lbug-interrupted-checkpoint-recovery` integration test. -Managed by `repo-manager.ts`. +Managed by `repo-manager.ts`. Publication reclaims unreferenced private graphs +and job artifacts under the slot lock, preserving current jobs, model spaces, +legacy reuse sources and prepared publications. Busy reader files are retried on +a later publication. See [MIGRATION.md](MIGRATION.md#resumable-analysis-and-separate-vector-storage-3598) +before mixing binary versions or rolling back. ### Automatic parse-cache identity diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 82896c508..8750bc69c 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -44,13 +44,13 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Trigger:** Semantic search quality drops; `stats.embeddings` in the index metadata (`gitnexus.json` / legacy `meta.json`) is 0 after refresh. - **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected. -- **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. +- **Why:** Plain `analyze` preserves prior vectors in their artifact (or restores the legacy in-graph cache); ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. ### Analyze finishes but embeddings are incomplete (partial embedding index) -- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["embedding-checkpoint-pending"]` (or the human-readable "Index incomplete reasons" line); `stats.embeddings` is honest and **non-zero**, and the preceding analyze log showed a `Warning: N node(s) lost their embeddings to embedding-endpoint failures` line (#2790). -- **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. -- **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. +- **Trigger:** `gitnexus status --json` reports `embedding-job-pending` (or the legacy `embedding-checkpoint-pending`), or an embedding failure follows graph publication. +- **Do:** Re-run plain `gitnexus analyze`; retained work resumes without `--embeddings`. Inspect `embeddingProgress` for completed, reused, failed and excluded nodes. `--force` rebuilds the graph and retains compatible vectors for reuse; `--drop-embeddings` deliberately abandons embedding work. +- **Why:** Graph and vector publication are separate. The graph is usable while a job is pending; semantic search reports degradation until a sealed compatible artifact is published. Deterministic input rejections are retried across at most three invocations, then excluded with a recorded reason. Provider outages remain resumable. Do not delete job artifacts to clear a status warning: they contain paid vectors. ### Scope extraction is incomplete diff --git a/MIGRATION.md b/MIGRATION.md index 36a899840..f979f2658 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,5 +1,57 @@ # Migration Guide +## Resumable analysis and separate vector storage (#3598) + +Analysis with embeddings publishes a private `lbug.generation.` graph and +stores resumable vector artifacts under `embeddings//`. Metadata names +the active graph and sealed vector generation. Once a slot uses this layout, +plain analysis, watch and hooks continue using it. Pending embedding work resumes +on the next plain `gitnexus analyze`; `--force` rebuilds the graph while retaining +compatible vectors for reuse. `--drop-embeddings` explicitly abandons embeddings. + +Upgrade every CLI, MCP/serve process and installed hook that accesses the slot. +Restart long-running clients and rerun `gitnexus setup` with the upgraded binary. +Older versions do not understand the private graph pointer or separate vectors. +Do not run an older analyzer against the migrated slot. + +Before upgrading, stop readers and writers and back up the complete storage +slot, including metadata, graph files, `embeddings/`, and any referenced shared +store. To roll back, stop clients again and restore that complete backup with +the matching older binaries. Renaming a generation file to `lbug` does not move +vectors back into it. There is no reverse conversion for separate vector +artifacts; without a backup, rebuilding with an older version requires generating +embeddings again and can incur provider charges. + +A graph can be ready while embeddings remain pending. Embedding failure after +graph publication exits nonzero; the graph remains usable and the next analyze +resumes paid work. HTTP clients should read `embeddingProgress` and capability +reasons rather than relying on the former terminal SSE `partial` field. After +three deterministic input rejections, the job publishes healthy nodes and records +`excludedNodes` with reason `input-rejected`; status reports the omission. +On Windows, a legacy reader can prevent retirement of `lbug`. Stop that reader +and rerun analyze; the prepared publication and vector artifacts are retained. + +Legacy source receipts now include the file's change time as well as its size +and modification time. Receipts from earlier previews without that guard are +retained but cannot certify reuse. Keep them with the original source when +recovering from a backup or re-exporting a verified source; do not fill in a +missing guard from the current file, which may have changed since retirement. + +`GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` is no longer accepted by analysis: it asks +the native engine to allocate an unbounded pool. Remove this setting from CLI, +server and auto-sync environments to use automatic sizing, or set a positive +byte limit. An explicit `--resource-budget` must also fit the page-size-scaled +COPY floor; increase the budget if admission rejects it. + +If a superseded or unverifiable `analysis-publication.json` blocks recovery, +stop every reader and writer and back up the complete storage slot first. +Move only that receipt into the backup, outside the slot, then run +`gitnexus analyze --force`. Keep the staged graphs and embedding artifacts until +the replacement graph and vectors are verified. A valid receipt should normally +be recovered by plain analysis; discarding it abandons its prepared graph work. +For an interrupted in-place write, keep clients stopped until the forced rebuild +completes; removing the receipt does not make the old graph healthy. + ## LadybugDB 0.22.0 storage and extensions LadybugDB 0.22.0 reads indexes written by 0.21.1, but a write/checkpoint upgrades diff --git a/RUNBOOK.md b/RUNBOOK.md index d65aeb4c4..512406278 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -129,13 +129,15 @@ costs; compare complete-source candidates separately from older capped inputs. **Important:** If you already had embeddings, a plain `npx gitnexus analyze` **preserves** them (Non-negotiable 5 in [GUARDRAILS.md](GUARDRAILS.md)) — pass `--embeddings` when you also want vectors generated for new or changed nodes, and `--drop-embeddings` only for a deliberate wipe. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none) — but that figure isn't always freshly measured: if a run's embedding-count query can't answer, it carries the previous run's number forward instead of writing a wrong zero. For a certified read, check `capabilities.vectorSearch.status` instead — it reads `unavailable` (never a stale count) whenever GitNexus can't vouch for the live vector index. -**Partial embedding index (analyze exits 0, but some nodes never got embedded):** A long run against a flaky embedding endpoint can finish successfully while a bounded number of sub-batches still fail. Affected nodes are dropped to zero rows (never left half-written) and recorded as a pending `embeddingCheckpoint`; `npx gitnexus status` then reports `incompleteReasons: ["embedding-checkpoint-pending"]`. Recovery is a plain: +**Embedding work interrupted after graph publication:** The graph and keyword index become available before embedding work starts. An embedding failure makes analyze exit nonzero while keeping that graph available and retaining acknowledged embedding work. `npx gitnexus status --json` reports graph freshness separately from embedding progress and vector readiness. Resume with: ```bash npx gitnexus analyze ``` -No `--embeddings` flag needed — a retained checkpoint forces embedding generation for the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. +No `--embeddings` flag is needed for a retained job. `npx gitnexus embeddings sync` also resumes work against an already published graph. Compatible acknowledged units do not call the provider again; a unit interrupted before acknowledgement may replay. A source or embedding-configuration change revalidates reuse, and changing worker counts or memory budgets does not invalidate paid vectors. `--drop-embeddings` deliberately abandons retained vectors. `--force` rebuilds the graph while retaining compatible paid work for reuse. + +Older indexes may instead carry an `embeddingCheckpoint` and report `embedding-checkpoint-pending`. Plain analyze recovers that checkpoint, including any explicitly referenced staging data, before migrating it. Historical vectors without verified provider/model provenance are retained without being relabeled as output from the current model. **Collapsed graph write (analyze exits NON-ZERO and says INCOMPLETE):** A run can finish writing metadata while only a fraction of the relationships it produced are readable back from the index — edges collapsing to a small share of what was built, or a `CodeRelation` table that never materialized (which reads as a persisted count of zero). Because the metadata IS written and the DB does hold rows, nothing looks broken: queries answer with missing edges rather than an error, which is a confident empty answer rather than a failure. `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`, the analyze summary prints `Repository indexed INCOMPLETELY` with the expected and persisted counts, and the CLI exits non-zero so automation is not told an unusable index is fine. @@ -280,6 +282,20 @@ on a partial result. Analyze re-execs Node with a **large old-space heap** when needed (`analyze.ts`). If you still OOM on huge repos, close other processes, avoid `--embeddings` for a first pass, or analyze a smaller path if supported by your workflow. +`--resource-budget ` admits the analyzer process family within one memory +reservation: the frontend, main heap, parser workers, native graph/vector pools, +embedding model, bounded input queue, and headroom. For example: + +```bash +npx gitnexus analyze --embeddings --resource-budget 8192 --memory-budget 3000 --workers 2 +``` + +`--memory-budget` still controls only the main V8 heap. Explicit settings that do +not fit the family budget fail before work starts. Without a family override, +GitNexus derives reservations from available memory and CPUs. The budget controls +admission and queue sizes; it is not an operating-system RSS limit. Watch mode +accepts the same family budget and drains active work before shutdown. + `Unable to allocate memory` / `The buffer pool is full` during graph writes refers to LadybugDB's native buffer pool. Increasing `NODE_OPTIONS`, `--memory-budget`, or worker heap limits does not resize that pool. Recognized @@ -294,8 +310,11 @@ GITNEXUS_LBUG_BUFFER_POOL_SIZE=3221225472 npx gitnexus analyze --force The override bypasses automatic pool sizing. Budget for the Node heap, parse workers, other native allocations, and the OS alongside it; a larger pool can -exhaust the host. `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` restores LadybugDB's native -80%-of-RAM default, not an unlimited supply of memory. If memory is insufficient, +exhaust the host. Outside bounded analysis, `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` +selects LadybugDB's native 80%-of-RAM default. Analysis and embedding admission +reject zero because that native allocation cannot fit alongside their other +reservations. Unset the variable for automatic sizing, or set an explicit byte +limit of at least 8 MiB that fits the resource budget. If memory is insufficient, exclude generated/vendor directories or use a larger host. A measured graph-write collapse aborts a staged build before publication, diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 524fbd1ab..b6fd91bab 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -102,6 +102,7 @@ gitnexus analyze --max-processes # Process-detection process cap (replaces gitnexus analyze --max-entry-point-candidates # Ranked entry-point pool (default 200; raise when the warning names it) gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots gitnexus analyze --asyncapi-spec ./docs/asyncapi # Resolve broker addresses from AsyncAPI 3.x documents +gitnexus analyze --resource-budget 8192 # Strict process-family reservation in MiB gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and --max-old-space-size gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) @@ -284,7 +285,7 @@ For local model setup and switching instructions, see [Local Embedding Model](.. | `GITNEXUS_CONTENT_RETENTION` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | You need to reduce persisted source text while preserving graph structure. | | `GITNEXUS_SKIP_FTS` | unset | When exactly `1`, skips FTS extension loading and keyword index creation during analyze. Equivalent to `--skip-fts`; a later analyze without either option restores FTS. | Graph-only consumers with their own retrieval, or short-lived indexes that do not need keyword search. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded analysis requires a positive byte ceiling of at least 8 MiB; zero and malformed values fail preflight. Other readers retain the legacy zero/default policy. During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning, and invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | | `GITNEXUS_ATOMIC_INCREMENTAL` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. | A staged incremental analyze fails (see _Troubleshooting_ in the package README), or you need in-place writes. | | `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | @@ -381,3 +382,31 @@ flowchart TD ConnA -->|"queries"| RepoA ConnB -->|"queries"| RepoB ``` + +### Analysis resource admission and resumable status + +Automatic analysis retains the RAM-aware main heap and independent parser-worker +ceilings. These are virtual limits and may sum above physical RAM. Use +`--resource-budget ` for a strict phase reservation covering the supervisor, +main heap, workers, native pools, queues and embedding model. The two phases do +not overlap. `--memory-budget` alone changes the main heap without removing parser +workers; with a strict total budget, incompatible overrides fail before work. +Embedding thread settings are validated only when embeddings are requested or pending. + +`status --json` reports `index.graphStatus` for a checkout, +`index.graphIncompleteReasons`, `index.embeddingProgress` and `index.capabilities`. +Progress includes its phase, completed/reused/failed nodes and pending units; +`excludedNodes` records permanently rejected inputs. Stored metadata separately +names `graphGeneration`, `vectorGeneration` and `embeddingJob`. +A preserved vector generation may refer to an older graph. Pending jobs report +`embedding-job-pending`; a published artifact with omitted inputs reports +`embedding-inputs-rejected`. A ready graph does not imply ready semantic search. + +`GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` is an internal inherited reservation, in +nonnegative integer bytes, for the waiting parent process. Leave it unset for +automatic accounting; a supplied value is included in admission. It does not +set the parent's V8 heap limit. + +A local-model job can report "Embedding runtime is busy" while an existing query +owns the model. Retry after that query finishes; admission does not replace a +sidecar that still has physical requests in flight. diff --git a/gitnexus-claude-plugin/hooks/registry-query.cjs b/gitnexus-claude-plugin/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-claude-plugin/hooks/registry-query.cjs +++ b/gitnexus-claude-plugin/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus-cursor-integration/hooks/registry-query.cjs b/gitnexus-cursor-integration/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-cursor-integration/hooks/registry-query.cjs +++ b/gitnexus-cursor-integration/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus-factory-plugin/hooks/registry-query.cjs b/gitnexus-factory-plugin/hooks/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus-factory-plugin/hooks/registry-query.cjs +++ b/gitnexus-factory-plugin/hooks/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus/README.md b/gitnexus/README.md index d75c91a3c..64de67835 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -299,6 +299,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and any --max-old-space-size pin +gitnexus analyze --resource-budget 8192 # Process-family memory reservation in MiB, including workers, native pools and embedding work gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) gitnexus auto-sync [init|start|restart|stop|status|reset] # Scheduled remote clone/pull + analyze from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos @@ -778,7 +779,7 @@ Configure the behavior with these environment variables: | `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. | | `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores the native 80%-of-RAM default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | Native buffer-pool ceiling in bytes. Bounded analysis requires a positive integer of at least 8 MiB; zero and malformed values fail preflight. Query readers retain the legacy zero/default policy. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning. Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | | `GITNEXUS_ATOMIC_INCREMENTAL` | `0`, `1` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. | @@ -842,6 +843,22 @@ To set the main-thread heap yourself on a memory-constrained host, pass the auto-sizer and any `--max-old-space-size` pin. It sizes the main thread only; parse workers keep their own caps. +Use `--resource-budget ` to admit the complete analyzer process family, +including the frontend, main heap, workers, native pools, embedding model, input +queue, and headroom. An explicit heap or pool that cannot fit is rejected before +work starts. Automatic sizing retains independent RAM-aware main and worker +ceilings, whose sum may exceed RAM. Only an explicit resource budget imposes an +aggregate phase ceiling; this reservation is +not an operating-system RSS limit. `analyze --watch` accepts the same budget. + +Embedding-enabled analysis publishes the graph and keyword index first. If the +provider fails, analyze exits nonzero while those queries remain available. Run +plain `gitnexus analyze` or `gitnexus embeddings sync` to resume a retained job. +Compatible acknowledged work is reused without provider requests; an interrupted +unacknowledged unit may replay. Changing resource settings does not invalidate +paid vectors. `gitnexus status --json` exposes graph freshness and embedding +progress separately; semantic search uses a completed, validated vector artifact. + Escape hatches (`GITNEXUS_MEMORY=off` to decline the autopilot, `GITNEXUS_WORKER_HEAP_MB` to size workers yourself) are listed in the environment-variable table below — @@ -959,6 +976,7 @@ Four env vars expose the pool's resilience layers (respawn budget, cumulative-ti | `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. | | `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. | +| `GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` | nonnegative integer bytes | automatic | Internal inherited reservation for the waiting parent process. Leave unset for automatic accounting; a supplied value counts toward analysis admission and does not set the parent heap. | | `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. | | `GITNEXUS_HEAP_LIMIT_SOURCE` | unset | Set by analyze itself (`budget` or `auto`) to record where the heap limit came from, so out-of-memory advice points at `--memory-budget` rather than a `--max-old-space-size` pin. | Never — internal; set `--memory-budget` instead. | | `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. | diff --git a/gitnexus/bench/analyze-resume/README.md b/gitnexus/bench/analyze-resume/README.md new file mode 100644 index 000000000..048030339 --- /dev/null +++ b/gitnexus/bench/analyze-resume/README.md @@ -0,0 +1,61 @@ +# Resumable analyze benchmark + +Build two checkouts with matching dependencies on the same filesystem, then run +from the candidate's `gitnexus/` directory: + +```sh +node bench/analyze-resume/measure.mjs --check \ + --baseline /path/to/base/gitnexus/dist/cli/index.js \ + --output /tmp/analyze-resume-explicit.json +``` + +Repeat with `--default-resources` and a new output path to compare automatic +resource allocation. The explicit profile uses a 2 GiB heap, two parser workers, +and a 256 MiB native pool on both sides. Do not run indexing or tests alongside +these measurements. + +`baseline.json` records the corpus settings, seven measured pairs per scenario, +commit IDs, and original Linux/Node 22 results. Its historical timings are +provenance, not portable wall-clock limits. The runner compares the supplied base +and candidate in alternating order, after a warmup, on the current machine. + +The six CI scenarios cover graph-only analysis, warm no-op, an incremental edit, +fresh HTTP embeddings, cached HTTP embeddings, and retry after a deterministic +provider outage. Setup runs are outside the measured interval. Assertions require +equal graph/vector counts and embedding-input fingerprints, recorded runtime +dependencies, completed publication, and no paid requests for a cached run. + +`--check` requires all six scenarios, the recorded corpus size, a warmup and at +least seven pairs. It fails when the paired 95% interval lies entirely +above the candidate/base ratio of **1.0**. The interval encloses both the +percentile bootstrap and Student-t interval on paired log ratios, avoiding the +narrow bootstrap-only interval with seven samples. Student-t inference assumes +independent pairs with approximately normal log ratios +([NIST confidence interval definition](https://www.itl.nist.gov/div898/handbook/eda/section3/eda352.htm)). An interval crossing 1 remains +**inconclusive** and is reported as such; a successful CI gate means no slowdown +was statistically detected, not proof of equal performance. The original +graph-only interval in `baseline.json` is inconclusive and has not been discarded. + +`.github/workflows/ci-tests.yml` runs both resource profiles against the PR base +(the first parent on non-PR runs). Failures block the existing CI Gate. Each job +uses `--allow-dependency-changes` to measure the complete shipped runtimes, so +dependency updates and release-version changes are benchmarked as well. Reports +record both lockfile/native digests and each pair's resolved dependency receipts +and parity. Manual runs require identical dependencies unless this flag is set. +Each job runs `--validate-gate` decision checks before measuring and +uploads raw samples, summaries, dependency receipts and analyzer logs, including +partial results on failure. The runner never overwrites an existing report. + +The explicit-profile CI job also uploads `analyze-resume-cpu-profile-explicit` before the +full comparison. This separate one-pair fresh-HTTP run uses `--cpu-profile` to +capture Node CPU profiles for each process and parser worker. Profiling is +incompatible with `--check`; acceptance timings always run without profiling. + +Benchmark execution belongs exclusively to the dedicated CI jobs above. It is +not included in the unit or integration test suites. + +`--help` lists additional options, including optional local-model scenarios. +Synthetic fixtures do not establish query ranking, concurrent query latency, +local-model throughput, exact peak RSS or platform correctness. Reports keep +overall acceptance marked incomplete. SIGINT sent to the harness stops between +invocations and lets any active native CLI exit naturally. diff --git a/gitnexus/bench/analyze-resume/baseline.json b/gitnexus/bench/analyze-resume/baseline.json new file mode 100644 index 000000000..82f6ac52e --- /dev/null +++ b/gitnexus/bench/analyze-resume/baseline.json @@ -0,0 +1,256 @@ +{ + "schemaVersion": 1, + "maxPairedRatio": 1, + "settings": { + "pairs": 7, + "warmups": 1, + "modules": 64, + "functions": 16, + "heap-mb": 2048, + "workers": 2, + "sample-ms": 25 + }, + "reference": { + "measuredAt": "2026-11-05T19:17:51.910Z", + "baselineCommit": "a0586d967f391f6f78e24d8c7162abfe350ae195", + "candidateCommit": "8c1b489c962a223b77567c21dcd1a9f2c9788851", + "environment": { + "node": "v22.23.3", + "platform": "linux", + "architecture": "x64", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "resourceProfile": "explicit-heap-workers-pool" + }, + "note": "Historical samples are provenance, not portable wall-clock limits. CI compares the current PR base and candidate on the same runner. An interval crossing 1 remains inconclusive." + }, + "scenarios": { + "graph-only": { + "pairs": 7, + "baselineMedianMs": 11254.142376000003, + "candidateMedianMs": 10191.042332999998, + "geometricRatio": 1.0582904590478737, + "pairedBootstrap95": [0.9484949043112423, 1.2159205271380682], + "evidence": "inconclusive", + "samplesMs": [ + { + "baseline": 11254.142376000003, + "candidate": 10144.616939999996 + }, + { + "baseline": 10671.354456000001, + "candidate": 9944.314222 + }, + { + "baseline": 10188.076402000006, + "candidate": 10191.042332999998 + }, + { + "baseline": 11853.487859999994, + "candidate": 17919.62693900001 + }, + { + "baseline": 11739.811645999987, + "candidate": 14308.022194000005 + }, + { + "baseline": 10668.565815000009, + "candidate": 10129.661259000015 + }, + { + "baseline": 11425.665953999996, + "candidate": 11556.346957000002 + } + ] + }, + "noop": { + "pairs": 7, + "baselineMedianMs": 2184.29468500003, + "candidateMedianMs": 1950.1828749999986, + "geometricRatio": 0.8930352942528001, + "pairedBootstrap95": [0.8795970798771887, 0.905936077701467], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 2198.722995999997, + "candidate": 1915.772708000004 + }, + { + "baseline": 2239.4145290000015, + "candidate": 1939.7219499999774 + }, + { + "baseline": 2176.4498259999673, + "candidate": 1950.1828749999986 + }, + { + "baseline": 2184.29468500003, + "candidate": 2005.5672990000458 + }, + { + "baseline": 2179.4396279999637, + "candidate": 1966.6888930000132 + }, + { + "baseline": 2128.4910190000082, + "candidate": 1891.617927000043 + }, + { + "baseline": 2208.1911739999778, + "candidate": 2008.7850609999732 + } + ] + }, + "incremental": { + "pairs": 7, + "baselineMedianMs": 5783.258446000051, + "candidateMedianMs": 5541.312566999986, + "geometricRatio": 0.9674187960203702, + "pairedBootstrap95": [0.9561405967756264, 0.9816590666894027], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 5779.961653999984, + "candidate": 5672.744095000031 + }, + { + "baseline": 5806.2170550000155, + "candidate": 5541.312566999986 + }, + { + "baseline": 5861.75522000005, + "candidate": 5892.487050000054 + }, + { + "baseline": 5720.598826000001, + "candidate": 5503.345939000021 + }, + { + "baseline": 5744.461226000101, + "candidate": 5528.829626999912 + }, + { + "baseline": 5783.258446000051, + "candidate": 5528.050583000062 + }, + { + "baseline": 5888.105557000032, + "candidate": 5603.379379000049 + } + ] + }, + "http-fresh": { + "pairs": 7, + "baselineMedianMs": 15511.024734000093, + "candidateMedianMs": 14335.91775100003, + "geometricRatio": 0.9272010938503702, + "pairedBootstrap95": [0.918360962384628, 0.9365473751189902], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 15511.024734000093, + "candidate": 14335.91775100003 + }, + { + "baseline": 15510.167916000006, + "candidate": 14261.329035999952 + }, + { + "baseline": 15609.408870000043, + "candidate": 14153.163587999996 + }, + { + "baseline": 15409.513349999906, + "candidate": 14634.943147000042 + }, + { + "baseline": 15519.041560999933, + "candidate": 14481.790279999957 + }, + { + "baseline": 15767.326278000022, + "candidate": 14690.776563999942 + }, + { + "baseline": 15408.597242999938, + "candidate": 14267.204865999985 + } + ] + }, + "http-cached": { + "pairs": 7, + "baselineMedianMs": 2149.4189200000837, + "candidateMedianMs": 1919.7719739999156, + "geometricRatio": 0.8972216499596883, + "pairedBootstrap95": [0.8876158802568739, 0.9067963765721975], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 2165.3657219999004, + "candidate": 1909.4434619999956 + }, + { + "baseline": 2132.151501000044, + "candidate": 1941.053768999991 + }, + { + "baseline": 2141.9542270000093, + "candidate": 1934.437436999986 + }, + { + "baseline": 2155.985989000066, + "candidate": 1919.7719739999156 + }, + { + "baseline": 2113.9462319999, + "candidate": 1902.646522999974 + }, + { + "baseline": 2149.4189200000837, + "candidate": 1968.86528300005 + }, + { + "baseline": 2152.111167999916, + "candidate": 1892.6074149999768 + } + ] + }, + "http-resume": { + "pairs": 7, + "baselineMedianMs": 15869.910479000071, + "candidateMedianMs": 4004.7973079998046, + "geometricRatio": 0.2485479851486286, + "pairedBootstrap95": [0.24355429100174475, 0.25278581390854993], + "evidence": "no-detected-regression", + "samplesMs": [ + { + "baseline": 15779.565613000188, + "candidate": 4038.0054809998255 + }, + { + "baseline": 15869.910479000071, + "candidate": 4017.237259999849 + }, + { + "baseline": 15861.843901999993, + "candidate": 3971.7236019999254 + }, + { + "baseline": 15861.11537200003, + "candidate": 3973.9850310001057 + }, + { + "baseline": 17113.716924000066, + "candidate": 4044.0885320000816 + }, + { + "baseline": 16508.58412100002, + "candidate": 4004.7973079998046 + }, + { + "baseline": 15901.872792999959, + "candidate": 3999.7262199998368 + } + ] + } + } +} diff --git a/gitnexus/bench/analyze-resume/measure.mjs b/gitnexus/bench/analyze-resume/measure.mjs new file mode 100644 index 000000000..fe0511953 --- /dev/null +++ b/gitnexus/bench/analyze-resume/measure.mjs @@ -0,0 +1,824 @@ +/** Whole-command paired measurements. Run --help; never treats a smoke run as acceptance. */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn, execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { createRequire } from 'node:module'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +const SCENARIOS = [ + 'graph-only', + 'noop', + 'incremental', + 'http-fresh', + 'http-cached', + 'http-resume', + 'local-fresh', + 'local-cached', +]; +const sha = (text) => createHash('sha256').update(text).digest('hex'); + +async function runtimeEnvironment(cli) { + const describeFile = async (file) => { + const resolved = await fs.realpath(file); + const [stat, filesystem] = await Promise.all([ + fs.stat(resolved, { bigint: true }), + fs.statfs(resolved, { bigint: true }), + ]); + return { path: resolved, device: String(stat.dev), filesystem: String(filesystem.type) }; + }; + const nativeEntry = createRequire(cli).resolve('@ladybugdb/core'); + return { + cli: await describeFile(cli), + nativeEntry: await describeFile(nativeEntry), + nativeEntryDigest: sha(await fs.readFile(nativeEntry)), + lockDigest: sha(await fs.readFile(path.resolve(path.dirname(cli), '../../package-lock.json'))), + }; +} + +export function assertComparableRuntimes(baseline, candidate, allowDependencyChanges = false) { + for (const component of ['cli', 'nativeEntry']) { + assert.equal( + candidate[component].device, + baseline[component].device, + `${component} runtime files are on different devices; stage both checkouts on the same filesystem`, + ); + assert.equal( + candidate[component].filesystem, + baseline[component].filesystem, + `${component} runtime files are on different filesystem types`, + ); + } + if (!allowDependencyChanges) { + assert.equal(candidate.lockDigest, baseline.lockDigest, 'Dependency lockfiles differ'); + assert.equal( + candidate.nativeEntryDigest, + baseline.nativeEntryDigest, + 'Native runtime entry bytes differ', + ); + } +} + +export function assertCompletedCase(scenario, metadata, beforeMetadata) { + assert( + !metadata.incrementalInProgress && !metadata.graphWriteCollapsed, + 'Graph was not finalized', + ); + assert( + Number.isSafeInteger(metadata.stats?.nodes) && metadata.stats.nodes > 0, + 'Missing graph node count', + ); + if (scenario === 'noop') { + assert(beforeMetadata?.indexedAt, 'No-op setup lacks a freshness receipt'); + assert.equal(metadata.indexedAt, beforeMetadata.indexedAt, 'No-op changed graph freshness'); + } + if (scenario.startsWith('http-') || scenario.startsWith('local-')) { + assert(metadata.stats.embeddings > 0, 'Embedding scenario produced no vectors'); + assert( + !metadata.embeddingCheckpoint && !metadata.embeddingJob, + 'Embedding work remains pending', + ); + } +} +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +/** Paired log-ratio intervals. Crossing 1 is unresolved, never equivalence. */ +export function summarizePairs(pairs) { + assert(pairs.length > 0, 'At least one measured pair is required'); + const logs = pairs.map(({ baseline, candidate }) => { + assert( + Number.isFinite(baseline) && baseline > 0 && Number.isFinite(candidate) && candidate > 0, + ); + return Math.log(candidate / baseline); + }); + const average = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length; + let seed = 3598; + const random = () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return seed / 2 ** 32; + }; + const samples = Array.from({ length: 10_000 }, () => + Math.exp(average(logs.map(() => logs[Math.floor(random() * logs.length)]))), + ).sort((a, b) => a - b); + const bootstrap = [samples[250], samples[9749]]; + // Bootstrap alone understates uncertainty with seven pairs. Enclose it in + // the Student-t interval for the mean log ratio; round df down in this table. + const df = logs.length - 1; + const critical = [ + [1, 12.707], + [2, 4.303], + [3, 3.183], + [4, 2.777], + [5, 2.571], + [6, 2.447], + [7, 2.365], + [8, 2.307], + [9, 2.263], + [10, 2.229], + [15, 2.132], + [20, 2.086], + [30, 2.043], + ] + .filter(([degree]) => degree <= df) + .at(-1)?.[1]; + const mean = average(logs); + const margin = + df > 0 + ? critical * Math.sqrt(logs.reduce((sum, x) => sum + (x - mean) ** 2, 0) / df / logs.length) + : 0; + const student = [Math.exp(mean - margin), Math.exp(mean + margin)]; + const interval = [Math.min(bootstrap[0], student[0]), Math.max(bootstrap[1], student[1])]; + return { + pairs: pairs.length, + baselineMedianMs: median(pairs.map((p) => p.baseline)), + candidateMedianMs: median(pairs.map((p) => p.candidate)), + geometricRatio: Math.exp(average(logs)), + pairedBootstrap95: bootstrap, + pairedStudent95: student, + paired95: interval, + evidence: + pairs.length < 7 + ? 'smoke-only' + : interval[0] > 1 + ? 'regression' + : interval[1] < 1 + ? 'no-detected-regression' + : 'inconclusive', + }; +} + +/** Detect slowdowns against a contemporaneous base, never historical wall-clock timings. */ +export function checkPerformance(summaries, baseline) { + assert.equal(baseline.schemaVersion, 1, 'Unsupported benchmark baseline'); + assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression'); + const scenarios = Object.keys(baseline.scenarios); + assert(scenarios.length > 0, 'Missing baseline scenarios'); + const failedScenarios = []; + const inconclusiveScenarios = []; + for (const scenario of scenarios) { + const summary = summaries[scenario]; + assert(summary && summary.pairs >= 7, `Missing seven measured pairs for ${scenario}`); + const interval = summary.paired95; + assert( + Array.isArray(interval) && + interval.length === 2 && + interval.every((value) => Number.isFinite(value) && value > 0) && + interval[0] <= interval[1], + `Invalid paired interval for ${scenario}`, + ); + if (interval[0] > baseline.maxPairedRatio) failedScenarios.push(scenario); + else if (interval[1] >= baseline.maxPairedRatio) inconclusiveScenarios.push(scenario); + } + return { failedScenarios, inconclusiveScenarios }; +} + +/** Sample Linux process RSS once per process, including descendants, never per thread. */ +async function sampleTree(pid) { + if (process.platform !== 'linux') return undefined; + const pending = [pid]; + const seen = new Set(); + let bytes = 0; + while (pending.length) { + const current = pending.pop(); + if (seen.has(current)) continue; + seen.add(current); + try { + const [status, children] = await Promise.all([ + fs.readFile(`/proc/${current}/status`, 'utf8'), + fs.readFile(`/proc/${current}/task/${current}/children`, 'utf8'), + ]); + bytes += Number(status.match(/^VmRSS:\s+(\d+)\s+kB$/m)?.[1] ?? 0) * 1024; + pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number)); + } catch (error) { + if (!['ENOENT', 'ESRCH'].includes(error.code)) throw error; + } + } + return bytes; +} + +export async function measureProcess(command, args, { cwd, env, logFile, sampleMs = 25 }) { + const log = await fs.open(logFile, 'wx'); + let timer; + let polling; + let samplingError; + let peakRssBytes = process.platform === 'linux' ? 0 : null; + let samples = 0; + const start = performance.now(); + try { + const child = spawn(command, args, { cwd, env, stdio: ['ignore', log.fd, log.fd] }); + const poll = () => { + if (!polling && child.pid) { + polling = sampleTree(child.pid) + .then((rss) => { + if (rss !== undefined) { + peakRssBytes = Math.max(peakRssBytes, rss); + samples++; + } + }) + .catch((error) => { + samplingError ??= error.message; + }) + .finally(() => { + polling = undefined; + }); + } + }; + poll(); + timer = setInterval(poll, sampleMs); + const outcome = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (exitCode, signal) => resolve({ exitCode, signal })); + }); + const elapsedMs = performance.now() - start; + clearInterval(timer); + await polling; + return { + ...outcome, + elapsedMs, + sampledPeakProcessTreeRssBytes: samplingError ? null : peakRssBytes, + rssSamples: samples, + rssSampleIntervalMs: sampleMs, + samplingError, + logFile, + }; + } finally { + clearInterval(timer); + await polling; + await log.close(); + } +} + +export async function startProvider(dimensions = 256) { + let active; + const server = createServer(async (req, res) => { + const counters = active; + try { + assert(counters && req.method === 'POST', 'Unexpected provider request'); + let size = 0; + const chunks = []; + for await (const chunk of req) { + size += chunk.length; + assert(size <= 8 * 1024 * 1024, 'Benchmark request exceeds 8 MiB'); + chunks.push(chunk); + } + const { input } = JSON.parse(Buffer.concat(chunks).toString('utf8')); + assert( + Array.isArray(input) && input.length && input.every((text) => typeof text === 'string'), + ); + counters.requests++; + counters.requestBytes += size; + if (counters.acceptedInputs + input.length > counters.failAfter) { + counters.rejectedRequests++; + res.writeHead(503, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'Deterministic interrupted-run fixture' } })); + return; + } + const data = input.map((text, index) => { + const hash = createHash('sha256').update(text).digest(); + counters.inputs.add(hash.toString('hex')); + counters.acceptedInputs++; + // Same text produces exactly the same nonzero vector in either implementation. + const embedding = Array.from({ length: dimensions }, (_, i) => (hash[i % 32] + 1) / 256); + return { object: 'embedding', index, embedding }; + }); + const body = JSON.stringify({ object: 'list', data, model: 'gitnexus-benchmark' }); + counters.responseBytes += Buffer.byteLength(body); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(body); + } catch (error) { + if (counters) counters.error ??= error.message; + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: error.message } })); + } + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + return { + url: `http://127.0.0.1:${server.address().port}/v1`, + begin(failAfter = Infinity) { + active = { + failAfter, + requests: 0, + rejectedRequests: 0, + acceptedInputs: 0, + requestBytes: 0, + responseBytes: 0, + inputs: new Set(), + }; + return active; + }, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} + +function moduleText(moduleIndex, modules, functions, revision = 0) { + const previous = (moduleIndex + modules - 1) % modules; + const lines = [`import { task_${previous}_0 } from './module-${previous}.js';`]; + for (let i = 0; i < functions; i++) { + const dependency = i ? `task_${moduleIndex}_${i - 1}` : `task_${previous}_0`; + lines.push(`/** Compute deterministic module ${moduleIndex}, operation ${i}. */`); + lines.push(`export function task_${moduleIndex}_${i}(value: number): number {`); + lines.push(` if (value <= 0) return ${i + (moduleIndex === 0 && i === 0 ? revision : 0)};`); + lines.push(` return ${dependency}(value - 1) + ${i + 1};`, '}'); + } + return lines.join('\n') + '\n'; +} + +async function fixture(directory, modules, functions) { + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'package.json'), '{"type":"module"}\n'); + for (let i = 0; i < modules; i++) { + await fs.writeFile(path.join(directory, `module-${i}.ts`), moduleText(i, modules, functions)); + } + const git = (...args) => + execFileSync('git', args, { + cwd: directory, + stdio: 'pipe', + env: { + ...process.env, + GIT_AUTHOR_DATE: '2026-01-01T00:00:00Z', + GIT_COMMITTER_DATE: '2026-01-01T00:00:00Z', + }, + }); + git('init', '-q'); + git('config', 'user.name', 'GitNexus benchmark'); + git('config', 'user.email', 'benchmark@example.invalid'); + git('config', 'commit.gpgsign', 'false'); + const hooks = path.join(directory, '.git', 'empty-hooks'); + await fs.mkdir(hooks); + git('config', 'core.hooksPath', hooks); + git('add', '.'); + git('commit', '-qm', 'fixture'); + return git; +} + +function providerSnapshot(counters) { + return { + requests: counters.requests, + rejectedRequests: counters.rejectedRequests, + acceptedInputs: counters.acceptedInputs, + uniqueInputs: counters.inputs.size, + uniqueInputFingerprint: sha([...counters.inputs].sort().join('\n')), + requestBytes: counters.requestBytes, + responseBytes: counters.responseBytes, + error: counters.error, + }; +} + +async function runCase({ + cli, + scenario, + directory, + modules, + functions, + heapMb, + workers, + defaultResources, + cpuProfile, + provider, + sampleMs, + stopRequested, +}) { + const repo = path.join(directory, 'repo'); + const store = path.join(directory, 'store'); + const git = await fixture(repo, modules, functions); + const env = { ...process.env }; + // Isolate GitNexus configuration/registry/credentials; never read a user's real config. + for (const key of Object.keys(env)) if (key.startsWith('GITNEXUS_')) delete env[key]; + delete env.NODE_OPTIONS; + Object.assign(env, { + GITNEXUS_HOME: path.join(directory, 'home'), + GITNEXUS_STORAGE_PATH: store, + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_WORKER_READY_TIMEOUT_MS: '30000', + }); + if (!defaultResources) env.GITNEXUS_LBUG_BUFFER_POOL_SIZE = String(256 * 1024 * 1024); + const http = scenario.startsWith('http-'); + const embedding = http || scenario.startsWith('local-'); + if (http) + Object.assign(env, { + GITNEXUS_EMBEDDING_URL: provider.url, + GITNEXUS_EMBEDDING_MODEL: 'gitnexus-benchmark', + GITNEXUS_EMBEDDING_DIMS: '256', + GITNEXUS_EMBEDDING_API_KEY: 'benchmark-only', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '16', + }); + const args = [ + ...(cpuProfile ? ['--cpu-prof', `--cpu-prof-dir=${directory}`] : []), + cli, + 'analyze', + repo, + '--index-only', + ]; + if (!defaultResources) args.push('--memory-budget', String(heapMb), '--workers', String(workers)); + if (embedding) args.push('--embeddings', '0'); + const counters = provider.begin( + scenario === 'http-resume' ? Math.max(16, Math.floor((modules * functions) / 3)) : Infinity, + ); + const invoke = async (name, allowFailure = false) => { + assert( + !stopRequested(), + 'Benchmark interrupted between invocations; native children were not signalled', + ); + const result = await measureProcess(process.execPath, args, { + cwd: repo, + env, + logFile: path.join(directory, `${name}.log`), + sampleMs, + }); + if (!allowFailure) + assert.equal(result.exitCode, 0, `Analyze failed; inspect ${result.logFile}`); + assert(!result.signal, `Analyze ended with ${result.signal}; inspect ${result.logFile}`); + assert(!counters.error, counters.error); + return result; + }; + let setup; + if (['noop', 'incremental', 'http-cached', 'http-resume', 'local-cached'].includes(scenario)) { + setup = await invoke('setup', scenario === 'http-resume'); + if (scenario === 'http-resume') { + assert( + counters.rejectedRequests > 0 && counters.acceptedInputs > 0, + 'Resume setup must actually interrupt partially successful work', + ); + counters.failAfter = Infinity; + } + if (scenario === 'incremental') { + await fs.writeFile(path.join(repo, 'module-0.ts'), moduleText(0, modules, functions, 1)); + git('add', 'module-0.ts'); + git('commit', '-qm', 'one-function edit'); + } + } + const beforeMetadata = + setup && scenario !== 'http-resume' + ? JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8')) + : undefined; + const before = providerSnapshot(counters); + const measurement = await invoke('measured'); + const after = providerSnapshot(counters); + if (scenario === 'http-cached') + assert.equal(after.acceptedInputs, before.acceptedInputs, 'Cached run made paid requests'); + const metadata = JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8')); + assertCompletedCase(scenario, metadata, beforeMetadata); + const stats = metadata.stats; + const result = { + ...measurement, + setup, + command: [process.execPath, ...args], + providerBefore: before, + providerAfter: after, + measuredProviderRequests: after.requests - before.requests, + measuredProviderInputs: after.acceptedInputs - before.acceptedInputs, + equalWork: { + files: stats.files, + nodes: stats.nodes, + edges: stats.edges, + embeddings: stats.embeddings ?? 0, + uniqueInputs: after.uniqueInputs, + inputFingerprint: after.uniqueInputFingerprint, + }, + metadata: { + runnerIdentity: metadata.runnerIdentity, + graphGeneration: metadata.graphGeneration, + vectorGeneration: metadata.vectorGeneration, + embeddingCheckpoint: metadata.embeddingCheckpoint, + embeddingJob: metadata.embeddingJob, + }, + }; + await fs.writeFile( + path.join(directory, 'measurement.json'), + JSON.stringify(result, null, 2) + '\n', + ); + return result; +} + +export async function main(argv = process.argv.slice(2)) { + const baseline = JSON.parse( + await fs.readFile(new URL('./baseline.json', import.meta.url), 'utf8'), + ); + const { values: v } = parseArgs({ + args: argv, + options: { + baseline: { type: 'string' }, + candidate: { type: 'string' }, + output: { type: 'string' }, + scenarios: { type: 'string', default: Object.keys(baseline.scenarios).join(',') }, + pairs: { type: 'string', default: '7' }, + warmups: { type: 'string', default: '1' }, + modules: { type: 'string', default: '64' }, + functions: { type: 'string', default: '16' }, + 'heap-mb': { type: 'string', default: '2048' }, + workers: { type: 'string', default: '2' }, + 'default-resources': { type: 'boolean', default: false }, + 'cpu-profile': { type: 'boolean', default: false }, + 'allow-dependency-changes': { type: 'boolean', default: false }, + 'sample-ms': { type: 'string', default: '25' }, + 'keep-stores': { type: 'boolean', default: false }, + check: { type: 'boolean', default: false }, + 'validate-gate': { type: 'boolean' }, + help: { type: 'boolean' }, + }, + }); + assert( + !(v.check && v['cpu-profile']), + 'CPU profiles are diagnostic runs, not acceptance timings', + ); + if (v['validate-gate']) { + const fixture = { schemaVersion: 1, maxPairedRatio: 1, scenarios: { case: {} } }; + const summary = (ratios) => + summarizePairs(ratios.map((candidate) => ({ baseline: 1, candidate }))); + assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(1.1)) }, fixture), { + failedScenarios: ['case'], + inconclusiveScenarios: [], + }); + assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(0.9)) }, fixture), { + failedScenarios: [], + inconclusiveScenarios: [], + }); + const mixed = summary([0.8, 0.9, 0.95, 1, 1.05, 1.1, 1.2]); + assert.deepEqual(checkPerformance({ case: mixed }, fixture), { + failedScenarios: [], + inconclusiveScenarios: ['case'], + }); + assert(mixed.paired95[0] <= mixed.pairedBootstrap95[0]); + assert(mixed.paired95[1] >= mixed.pairedBootstrap95[1]); + assert.throws(() => checkPerformance({ case: summary([1, 1]) }, fixture)); + assert.throws(() => summarizePairs([{ baseline: 0, candidate: 1 }])); + const runtime = { + cli: { device: '1', filesystem: '1' }, + nativeEntry: { device: '1', filesystem: '1' }, + lockDigest: 'a', + nativeEntryDigest: 'a', + }; + const changed = { ...runtime, lockDigest: 'b', nativeEntryDigest: 'b' }; + assert.throws(() => assertComparableRuntimes(runtime, changed)); + assertComparableRuntimes(runtime, changed, true); + assert.throws(() => + assertComparableRuntimes( + runtime, + { ...changed, cli: { device: '2', filesystem: '1' } }, + true, + ), + ); + console.log('Paired gate decision checks passed'); + return; + } + if (v.help) { + console.log(`Usage: node bench/analyze-resume/measure.mjs --baseline /repo/gitnexus/dist/cli/index.js --output /tmp/results.json + --candidate PATH Defaults to this checkout's built CLI + --scenarios LIST ${SCENARIOS.join(',')} + --pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only + --modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25 + --default-resources Omit heap, worker and native-pool overrides on both sides + --cpu-profile Retain per-process/thread CPU profiles; incompatible with --check + --allow-dependency-changes Compare shipped runtimes, including intentional dependency changes + --validate-gate Check decision logic without running benchmarks + --check Require the baseline corpus/scenarios and reject detected slowdowns + --keep-stores Retain owned index/home artifacts (logs and measurements always retained) +Includes full CLI startup, graph/manifest/native writes and final index work. +HTTP resume uses a deterministic provider outage, not an unsafe native-process kill. +RSS is sampled Linux process-family RSS; other platforms report it unavailable. +Local scenarios use the installed default model; prewarm its download before timing. +Live-query latency and platform cases remain separate acceptance requirements.`); + return; + } + assert(v.baseline && v.output, '--baseline and --output are required'); + const numbers = {}; + for (const key of [ + 'pairs', + 'warmups', + 'modules', + 'functions', + 'heap-mb', + 'workers', + 'sample-ms', + ]) { + numbers[key] = Number(v[key]); + assert( + Number.isSafeInteger(numbers[key]) && numbers[key] >= (key === 'warmups' ? 0 : 1), + `Invalid --${key}`, + ); + } + assert(numbers.modules >= 2, '--modules must be at least 2'); + const scenarios = v.scenarios.split(','); + assert( + scenarios.length && + new Set(scenarios).size === scenarios.length && + scenarios.every((x) => SCENARIOS.includes(x)), + 'Invalid or repeated scenario', + ); + if (v.check) { + assert(numbers.pairs >= 7 && numbers.warmups >= 1, '--check requires seven pairs and a warmup'); + for (const key of ['modules', 'functions']) + assert.equal(numbers[key], baseline.settings[key], `--check requires baseline ${key}`); + assert( + Object.keys(baseline.scenarios).every((scenario) => scenarios.includes(scenario)), + '--check requires all baseline scenarios', + ); + assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression'); + } + const candidate = + v.candidate ?? fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url)); + const clis = { baseline: path.resolve(v.baseline), candidate: path.resolve(candidate) }; + for (const cli of Object.values(clis)) await fs.access(cli); + const runtime = { + baseline: await runtimeEnvironment(clis.baseline), + candidate: await runtimeEnvironment(clis.candidate), + }; + assertComparableRuntimes(runtime.baseline, runtime.candidate, v['allow-dependency-changes']); + const output = path.resolve(v.output); + await fs.mkdir(path.dirname(output), { recursive: true }); + // Never overwrite a prior result; evidence files and temporary directories are exclusively owned. + const resultFile = await fs.open(output, 'wx'); + const work = await fs.mkdtemp(path.join(path.dirname(output), 'analyze-pairs-')); + const provider = await startProvider(); + let interrupted = false; + const interrupt = () => { + interrupted = true; + console.error( + 'Stopping between invocations; waiting for the active native CLI to exit normally.', + ); + }; + process.on('SIGINT', interrupt); + const report = { + schemaVersion: 1, + createdAt: new Date().toISOString(), + work, + environment: { + node: process.version, + platform: process.platform, + architecture: process.arch, + cpu: os.cpus()[0]?.model, + availableParallelism: os.availableParallelism(), + totalMemory: os.totalmem(), + settings: numbers, + resourceProfile: v['default-resources'] ? 'default' : 'explicit-heap-workers-pool', + cpuProfile: v['cpu-profile'], + comparison: v['allow-dependency-changes'] ? 'shipped-runtimes' : 'identical-dependencies', + runtime, + clis: {}, + }, + limitations: [ + 'Synthetic corpus only; stats and deterministic input coverage are checked, not query ranking.', + 'Linux RSS sampling is a lower-bound observation, not a hard memory limit or exact peak.', + 'Disk, phase timings, native reservations and queue peaks are not collected; optional CPU profiles are diagnostic only.', + 'Only the selected scenarios and resource profile were measured; live query latency and Windows require additional validation.', + ], + acceptance: 'incomplete', + samples: [], + summaries: {}, + }; + const save = async () => { + const bytes = Buffer.from(JSON.stringify(report, null, 2) + '\n'); + await resultFile.write(bytes, 0, bytes.length, 0); + await resultFile.truncate(bytes.length); + await resultFile.sync(); + }; + try { + for (const [side, cli] of Object.entries(clis)) { + report.environment.clis[side] = { + path: cli, + entryDigest: sha(await fs.readFile(cli)), + commit: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: path.dirname(cli), + encoding: 'utf8', + }).trim(), + }; + } + await save(); + for (const scenario of scenarios) { + const pairs = []; + for (let iteration = -numbers.warmups; iteration < numbers.pairs; iteration++) { + const ordinal = iteration + numbers.warmups; + const order = ordinal % 2 === 0 ? ['baseline', 'candidate'] : ['candidate', 'baseline']; + const pair = { scenario, iteration, warmup: iteration < 0, order }; + report.samples.push(pair); + for (const side of order) { + assert( + !interrupted, + 'Benchmark interrupted between invocations; partial evidence retained', + ); + const directory = path.join(work, `${scenario}-${ordinal}-${side}`); + pair[side] = await runCase({ + cli: clis[side], + scenario, + directory, + modules: numbers.modules, + functions: numbers.functions, + heapMb: numbers['heap-mb'], + workers: numbers.workers, + defaultResources: v['default-resources'], + cpuProfile: v['cpu-profile'], + sampleMs: numbers['sample-ms'], + provider, + stopRequested: () => interrupted, + }); + await save(); + console.log( + JSON.stringify({ + scenario, + iteration, + side, + elapsedMs: pair[side].elapsedMs, + requests: pair[side].measuredProviderRequests, + rss: pair[side].sampledPeakProcessTreeRssBytes, + }), + ); + if (!v['keep-stores']) { + // Only paths created inside this invocation's mkdtemp root are removed. + await fs.rm(path.join(directory, 'store'), { recursive: true, force: true }); + await fs.rm(path.join(directory, 'home'), { recursive: true, force: true }); + } + } + assert.deepEqual( + pair.candidate.equalWork, + pair.baseline.equalWork, + `Unequal work for ${scenario}; inspect retained measurements`, + ); + if (scenario === 'http-resume') { + assert.equal( + pair.candidate.providerAfter.acceptedInputs, + pair.candidate.providerAfter.uniqueInputs, + 'Resume replayed already accepted embedding inputs', + ); + assert.equal( + pair.candidate.measuredProviderInputs, + pair.candidate.providerAfter.uniqueInputs - + pair.candidate.providerBefore.acceptedInputs, + 'Resume must request only unfinished embedding inputs', + ); + } + const baselineDependencies = + pair.baseline.metadata.runnerIdentity?.dependencyRuntime?.digest; + assert( + /^sha256:[a-f0-9]{64}$/.test(baselineDependencies ?? ''), + 'Baseline dependency receipt missing', + ); + const candidateDependencies = + pair.candidate.metadata.runnerIdentity?.dependencyRuntime?.digest; + assert( + /^sha256:[a-f0-9]{64}$/.test(candidateDependencies ?? ''), + 'Candidate dependency receipt missing', + ); + pair.dependenciesMatch = candidateDependencies === baselineDependencies; + if (!v['allow-dependency-changes']) + assert( + pair.dependenciesMatch, + 'Resolved dependency payloads differ; use --allow-dependency-changes to compare shipped runtimes', + ); + if (iteration >= 0) + pairs.push({ baseline: pair.baseline.elapsedMs, candidate: pair.candidate.elapsedMs }); + } + report.summaries[scenario] = summarizePairs(pairs); + await save(); + } + if (v.check) { + report.check = checkPerformance(report.summaries, baseline); + await save(); + assert.equal( + report.check.failedScenarios.length, + 0, + `Performance regression: ${report.check.failedScenarios.join(', ')}; inspect ${output}`, + ); + if (report.check.inconclusiveScenarios.length) + console.warn( + `Inconclusive timings: ${report.check.inconclusiveScenarios.join(', ')}; not proof of equal performance.`, + ); + } + console.log( + JSON.stringify( + { output, summaries: report.summaries, check: report.check, acceptance: report.acceptance }, + null, + 2, + ), + ); + } catch (error) { + report.error = error.message; + await save(); + throw error; + } finally { + process.removeListener('SIGINT', interrupt); + await provider.close(); + await resultFile.close(); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/gitnexus/hooks/claude/registry-query.cjs b/gitnexus/hooks/claude/registry-query.cjs index 52abc3474..6d3b91abb 100644 --- a/gitnexus/hooks/claude/registry-query.cjs +++ b/gitnexus/hooks/claude/registry-query.cjs @@ -304,10 +304,21 @@ function isDirectChild(parent, child) { } // Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a -// shared-store checkout slot may read a commit graph in the same store -// instead of owning /lbug. Any other recorded value is ignored. +// slot may read an owned private generation; shared checkout slots may also +// read a commit graph in the same store. Other recorded values are ignored. function resolveGraphPath(storagePath, metadata) { const own = path.join(storagePath, LBUG_DIRECTORY); + const recorded = metadata && metadata.graphPath; + if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + } catch (error) { + if (!error || error.code !== 'ENOENT') return null; + } + return graph; + } const storesRoot = path.resolve( process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'), 'stores', @@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) { const root = path.dirname(checkoutsDir); if (path.basename(checkoutsDir) !== 'checkouts') return own; if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own; - const recorded = metadata && metadata.graphPath; if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); // Only a published `-` dir, never `.publish-*` staging. @@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) { const branchDir = branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null; const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath; + const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir); + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) return null; return { path: current, storagePath, - lbugPath: path.join(indexDir, LBUG_DIRECTORY), - metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir), + lbugPath, + metadata: indexMetadata, }; } } @@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) { const indexDir = branchIsIndexed ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : storagePath; + const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata; + const lbugPath = resolveGraphPath(indexDir, indexMetadata); + if (!lbugPath) continue; if (registeredPath.length > bestLen) { bestLen = registeredPath.length; best = { path: entry.path, storagePath, - lbugPath: branchIsIndexed - ? path.join(indexDir, LBUG_DIRECTORY) - : resolveGraphPath(storagePath, ownershipMetadata), - metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata, + lbugPath, + metadata: indexMetadata, }; } } diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index bcae19f36..0e4c92cc5 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -26,6 +26,15 @@ // Platform-specific logic tests — contain explicit process.platform guards // or test behavior that differs across operating systems const PLATFORM_LOGIC = [ + 'test/unit/embedding-sidecar-lifetime.test.ts', + 'test/unit/analysis-publication.test.ts', + 'test/unit/analysis-phase-supervisor.test.ts', + 'test/unit/shared-vector-artifact.test.ts', + 'test/unit/embedding-artifact.test.ts', + 'test/unit/embedding-job-progress.test.ts', + // Descriptor validation differs when O_NOFOLLOW is unavailable on Windows. + 'test/unit/embedding-job-file-security.test.ts', + 'test/unit/shared-store-read-security.test.ts', 'test/unit/setup.test.ts', 'test/unit/setup-jsonc.test.ts', 'test/unit/setup-codex.test.ts', @@ -164,6 +173,17 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/analysis-embedding-flow.test.ts', + 'test/integration/analysis-generation-routing.test.ts', + 'test/integration/analysis-publication-flow.test.ts', + 'test/integration/embedding-job-controller.test.ts', + 'test/integration/embedding-job-restart.test.ts', + 'test/integration/embedding-native-release.test.ts', + 'test/integration/legacy-embedding-controller.test.ts', + 'test/integration/legacy-vector-source.test.ts', + 'test/integration/embedding-vector-store.test.ts', + 'test/integration/legacy-vector-reuse-index.test.ts', + 'test/integration/legacy-vector-export.test.ts', 'test/integration/java-overload-impact-persistence.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/impact-private-type-file-seed.test.ts', diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index 2083487e1..79e3c753f 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -127,6 +127,8 @@ export interface AnalyzeOptions { * Integer, minimum 200; CLI-only (not a `.gitnexusrc` key). */ memoryBudget?: string; + /** Total process-family reservation, separate from the main V8 heap flag. */ + resourceBudget?: string; /** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */ workers?: string; /** Process-detection process cap. Positive integer string; `0` is invalid. */ diff --git a/gitnexus/src/cli/analyze-phase-ipc.ts b/gitnexus/src/cli/analyze-phase-ipc.ts new file mode 100644 index 000000000..f81901134 --- /dev/null +++ b/gitnexus/src/cli/analyze-phase-ipc.ts @@ -0,0 +1,197 @@ +/** Bounded CLI graph handoff. Live graph consumers run before this is sent. */ +import path from 'node:path'; +import type { AnalyzeResult } from '../core/run-analyze.js'; +import { + projectPreparedAnalysisHandoff, + type PreparedAnalysisHandoff, +} from '../core/analysis-phase-supervisor.js'; +import { FTS_SKIP_REASONS } from '../core/search/fts-policy.js'; + +export interface CliPreparedAnalysisMessage { + type: 'graph-prepared'; + handoff: PreparedAnalysisHandoff; + result: Pick< + AnalyzeResult, + | 'repoPath' + | 'repoName' + | 'storagePath' + | 'stats' + | 'ftsSkipped' + | 'ftsSkipReason' + | 'isPrimaryBranch' + >; + startedAt: number; + /** Snapshot from BEFORE graph/context generation, never recomputed on resume. */ + selfCommitSafety?: { agentsMd: boolean; claudeMd: boolean }; +} + +export interface CliAnalysisContinuation { + type: 'continue-analysis'; + writerPid: number; + prepared: CliPreparedAnalysisMessage; +} + +/** Validate both directions; never retain extra fields from an IPC object. */ +export function parseCliPreparedAnalysis(value: unknown): CliPreparedAnalysisMessage { + const input = value as CliPreparedAnalysisMessage; + if (!input || input.type !== 'graph-prepared' || !input.result) + throw new Error('Invalid CLI graph phase message'); + const safety = input.selfCommitSafety; + if (safety && (typeof safety.agentsMd !== 'boolean' || typeof safety.claudeMd !== 'boolean')) + throw new Error('Invalid CLI self-commit snapshot'); + return projectCliPreparedAnalysis( + { ...input.result, preparedGraph: input.handoff, rebuildReasons: [] }, + input.handoff.repoPath, + input.startedAt, + safety + ? new Map([ + ['AGENTS.md', safety.agentsMd], + ['CLAUDE.md', safety.claudeMd], + ]) + : undefined, + ); +} + +export async function receiveCliAnalysisContinuation( + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!process.connected || !process.send) + throw new Error('Analysis continuation requires its connected supervisor'); + return new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + process.removeListener('message', onMessage); + process.removeListener('disconnect', onDisconnect); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); + reject(signal?.reason ?? new Error('Analysis cancelled')); + }; + const onDisconnect = () => { + cleanup(); + reject(new Error('Analysis supervisor disconnected')); + }; + const onMessage = (message: unknown) => { + cleanup(); + try { + const input = message as CliAnalysisContinuation; + if ( + input?.type !== 'continue-analysis' || + !Number.isSafeInteger(input.writerPid) || + input.writerPid < 1 + ) + throw new Error('Invalid CLI continuation'); + resolve({ + type: 'continue-analysis', + writerPid: input.writerPid, + prepared: parseCliPreparedAnalysis(input.prepared), + }); + } catch (error) { + reject(error); + } + }; + const timer = setTimeout(() => { + cleanup(); + reject(new Error('Analysis supervisor did not send its continuation')); + }, 30_000); + process.once('message', onMessage); + process.once('disconnect', onDisconnect); + signal?.addEventListener('abort', onAbort, { once: true }); + process.send!({ type: 'analysis-continuation-ready' }, (error) => { + if (error) { + cleanup(); + reject(error); + } + }); + }); +} + +export function projectCliPreparedAnalysis( + result: AnalyzeResult, + repoPath: string, + startedAt: number, + selfCommitSafety?: ReadonlyMap, +): CliPreparedAnalysisMessage { + const handoff = projectPreparedAnalysisHandoff(result.preparedGraph); + if ( + result.graphWriteCollapsed || + result.alreadyUpToDate || + result.ftsRepairedOnly || + result.repoName !== handoff.repoName || + path.resolve(result.repoPath) !== path.resolve(repoPath) || + path.resolve(result.repoPath) !== path.resolve(handoff.repoPath) || + path.resolve(result.storagePath) !== path.resolve(handoff.storagePath) + ) + throw new Error('Prepared CLI graph does not match the requested analysis'); + if (!Number.isSafeInteger(startedAt) || startedAt < 0) + throw new Error('Invalid graph phase start time'); + + // Never spread stats or the result: either may gain corpus-sized fields. + const stats: AnalyzeResult['stats'] = {}; + for (const key of [ + 'files', + 'nodes', + 'edges', + 'communities', + 'processes', + 'embeddings', + ] as const) { + const value = result.stats[key]; + if (value === undefined) continue; + if (!Number.isSafeInteger(value) || value < 0) + throw new Error(`Invalid graph phase counter: ${key}`); + stats[key] = value; + } + for (const key of ['ftsSkipped', 'isPrimaryBranch'] as const) { + if (result[key] !== undefined && typeof result[key] !== 'boolean') + throw new Error(`Invalid graph phase flag: ${key}`); + } + if (result.ftsSkipReason !== undefined && !FTS_SKIP_REASONS.includes(result.ftsSkipReason)) + throw new Error('Invalid graph phase FTS reason'); + return { + type: 'graph-prepared', + handoff, + result: { + repoPath: handoff.repoPath, + repoName: handoff.repoName, + storagePath: handoff.storagePath, + stats, + ftsSkipped: result.ftsSkipped, + ftsSkipReason: result.ftsSkipReason, + isPrimaryBranch: result.isPrimaryBranch, + }, + startedAt, + selfCommitSafety: selfCommitSafety + ? { + agentsMd: selfCommitSafety.get('AGENTS.md') === true, + claudeMd: selfCommitSafety.get('CLAUDE.md') === true, + } + : undefined, + }; +} + +interface SupervisorChannel { + connected?: boolean; + send?: (message: CliPreparedAnalysisMessage, callback: (error: Error | null) => void) => unknown; +} + +export function requireCliAnalysisSupervisor(channel: SupervisorChannel = process) { + if (typeof channel.send !== 'function' || channel.connected !== true) + throw new Error('Prepared graph requires a connected analysis supervisor'); + return channel.send.bind(channel); +} + +/** A successful send is only delivery, never graph publication or process exit. */ +export async function sendCliPreparedAnalysis( + message: CliPreparedAnalysisMessage, + channel: SupervisorChannel = process, +): Promise { + const send = requireCliAnalysisSupervisor(channel); + await new Promise((resolve, reject) => { + // Await the callback before the caller exits. A false return means channel + // backpressure, not failure; the callback remains the delivery boundary. + send(message, (error: Error | null) => (error ? reject(error) : resolve())); + }); +} diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index a5f19d4e1..7a56ba7d1 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -1,6 +1,7 @@ /** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */ import path from 'node:path'; import fs from 'node:fs/promises'; +import { getHeapStatistics } from 'node:v8'; import { watchFile, unwatchFile } from 'node:fs'; import { watch, type FSWatcher } from 'chokidar'; import { createWatchIgnorePredicate } from '../config/ignore-service.js'; @@ -23,9 +24,14 @@ import { } from './analyze-config.js'; import { logger } from '../core/logger.js'; import type { AnalyzeOptions } from './analyze-options.js'; -import { ensureHeap } from './analyze.js'; +import { ensureHeap, usesManualAnalysisHeap } from './analyze.js'; import { cliError, cliInfo, cliWarn } from './cli-message.js'; import { parseIntegerOption } from './int-option.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; import { formatInvalidProcessDetectionOverride, parseProcessDetectionBudgetStrings, @@ -188,6 +194,18 @@ export async function resolveWatchOptions( cliWarn(formatInvalidProcessDetectionOverride(flag, raw)); }, ); + const resources = resolveHostAnalysisResources({ + totalMb: positiveInteger(merged.resourceBudget, '--resource-budget'), + mainHeapMb: + positiveInteger(cli.memoryBudget, '--memory-budget') ?? + Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + workers: workerPoolSize, + includeEmbeddings: await analysisMayEmbed(repoPath, { branch }), + supervisorBytes: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES), + }); setEnvironment( 'GITNEXUS_MAX_FILE_SIZE', @@ -201,6 +219,8 @@ export async function resolveWatchOptions( setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); return { + resources, + graphPublication: 'auto', pdg: merged.pdg, skipFts: merged.skipFts, branch, @@ -453,182 +473,220 @@ export async function watchCommandWithRunnerIdentity( inputPath?: string, cliOptions: WatchCliOptions = {}, ): Promise { - if ( - await ensureHeap({ cleanForwardedTermination: true, memoryBudget: cliOptions.memoryBudget }) - ) { - return; - } - - const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); - if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { - cliError(' gitnexus analyze --watch requires a Git repository.'); - process.exitCode = 1; - return; - } - const repoPath = await fs.realpath(requestedRepoPath); - try { - await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS); - } catch (error) { - cliError(` ${error instanceof Error ? error.message : String(error)}`); - process.exitCode = 1; - return; - } - const baselineEnvironment: WatchEnvironmentBaseline = { - maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, - workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, - verbose: process.env.GITNEXUS_VERBOSE, + let cancelled = false; + let stopWatching: (() => void) | undefined; + const cancel = () => { + cancelled = true; + stopWatching?.(); }; + const onControlMessage = (message: unknown) => { + if ((message as { type?: unknown } | null)?.type === 'cancel') cancel(); + }; + process.on('message', onControlMessage); + process.once('disconnect', cancel); try { - let ignoredConfigSignature: string | undefined; - const reportIgnoredConfig = (names: readonly string[]) => { - const signature = [...names].sort().join(','); - if (signature === ignoredConfigSignature) return; - ignoredConfigSignature = signature; - if (names.length > 0) { - cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); - } - }; - let debounceMs: number; - let analyzeOptions: CoreAnalyzeOptions; + if (process.send) { + if (!process.connected) cancel(); + else + process.send({ type: 'analysis-watch-ready' }, (error) => { + if (error) cancel(); + }); + } + const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { + cliError(' gitnexus analyze --watch requires a Git repository.'); + process.exitCode = 1; + return; + } + const repoPath = await fs.realpath(requestedRepoPath); try { - debounceMs = - positiveInteger( - cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), - '--debounce', - MAX_TIMER_DELAY_MS, - ) ?? DEFAULT_DEBOUNCE_MS; - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); + const manualHeap = usesManualAnalysisHeap(cliOptions.memoryBudget); + const admission = resolveHostAnalysisResources({ + totalMb: positiveInteger(cliOptions.resourceBudget, '--resource-budget'), + mainHeapMb: manualHeap + ? Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2) + : positiveInteger(cliOptions.memoryBudget, '--memory-budget'), + workers: positiveInteger(cliOptions.workers, '--workers'), + includeEmbeddings: await analysisMayEmbed(repoPath, { branch: cliOptions.branch }), + supervisorBytes: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES), + }); + if (cancelled) return; + if ( + !manualHeap && + (await ensureHeap({ + cleanForwardedTermination: true, + memoryBudget: cliOptions.memoryBudget, + admittedHeapMb: admission.mainHeapMb, + })) + ) + return; + await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS); } catch (error) { cliError(` ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; return; } - - let stopWatching!: () => void; - const stopped = new Promise((resolve) => { - stopWatching = resolve; - }); - const stop = () => stopWatching(); - process.once('SIGINT', stop); - process.once('SIGTERM', stop); + const baselineEnvironment: WatchEnvironmentBaseline = { + maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, + workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, + verbose: process.env.GITNEXUS_VERBOSE, + }; try { - let loop: WatchFileLoop; - let fatalRefreshError: unknown; - let configControlValid = true; - let lastSuccessfulRefreshAt: string | undefined; + let ignoredConfigSignature: string | undefined; + const reportIgnoredConfig = (names: readonly string[]) => { + const signature = [...names].sort().join(','); + if (signature === ignoredConfigSignature) return; + ignoredConfigSignature = signature; + if (names.length > 0) { + cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); + } + }; + let debounceMs: number; + let analyzeOptions: CoreAnalyzeOptions; try { - loop = await startWatchFileLoop( + debounceMs = + positiveInteger( + cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), + '--debounce', + MAX_TIMER_DELAY_MS, + ) ?? DEFAULT_DEBOUNCE_MS; + analyzeOptions = await resolveWatchOptions( repoPath, - debounceMs, - async (paths) => { - // Initial validation precedes watcher setup. Read again after it is - // armed so a change during startup cannot leave stale options live. - if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) { - const retryingInvalidConfig = !configControlValid; - try { - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); - configControlValid = true; - } catch (error) { - configControlValid = false; - throw new WatchControlReloadError( - retryingInvalidConfig - ? new Error( - 'Configuration remains invalid; fix it before indexing more changes.', - { - cause: error, - }, - ) - : error, - ); - } - } - const startedAt = Date.now(); - const result = await runFullAnalysis( - repoPath, - analyzeOptions, - { - onProgress: () => {}, - onLog: - process.env.GITNEXUS_VERBOSE === '1' - ? (message) => cliInfo(` ${message}`) - : undefined, - }, - runnerIdentityAtBootstrap, - ); - lastSuccessfulRefreshAt = new Date().toISOString(); - if (paths.length === 0) { - cliInfo( - result.alreadyUpToDate - ? `Watching ${repoPath}; index is up to date.` - : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, - ); - } else { - cliInfo( - refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), - ); - } - }, - (error, paths) => { - const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; - if (shouldStopAfterWatchRefreshFailure(error, paths)) { - fatalRefreshError = error; - const guardTimeout = isIndexLockGuardTimeout(error); - cliError( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - (guardTimeout - ? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.' - : 'Watch mode is stopping because the live index may have been updated in place.'), - guardTimeout - ? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath } - : undefined, - ); - stopWatching(); - return; - } - const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; - cliWarn( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - `Retry scheduled; last success ${lastSuccess}.`, - ); - }, - (error) => { - fatalRefreshError = error; - cliError( - `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + - 'Watch mode is stopping.', - ); - stopWatching(); - }, - process.env[GITNEXUS_RC_ENV]?.trim() || undefined, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, ); } catch (error) { - cliError( - ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, - ); + cliError(` ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; return; } - await stopped; - await loop.close(); - if (fatalRefreshError !== undefined) process.exitCode = 1; + if (cancelled) return; + const stopped = new Promise((resolve) => { + stopWatching = resolve; + }); + const stop = cancel; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + let loop: WatchFileLoop; + let fatalRefreshError: unknown; + let configControlValid = true; + let lastSuccessfulRefreshAt: string | undefined; + try { + loop = await startWatchFileLoop( + repoPath, + debounceMs, + async (paths) => { + if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) { + const retryingInvalidConfig = !configControlValid; + try { + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + configControlValid = true; + } catch (error) { + configControlValid = false; + throw new WatchControlReloadError( + retryingInvalidConfig + ? new Error( + 'Configuration remains invalid; fix it before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + const startedAt = Date.now(); + const result = await runFullAnalysis( + repoPath, + analyzeOptions, + { + onProgress: () => {}, + onLog: + process.env.GITNEXUS_VERBOSE === '1' + ? (message) => cliInfo(` ${message}`) + : undefined, + }, + runnerIdentityAtBootstrap, + ); + lastSuccessfulRefreshAt = new Date().toISOString(); + if (paths.length === 0) { + cliInfo( + result.alreadyUpToDate + ? `Watching ${repoPath}; index is up to date.` + : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, + ); + } else { + cliInfo( + refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), + ); + } + }, + (error, paths) => { + const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; + if (shouldStopAfterWatchRefreshFailure(error, paths)) { + fatalRefreshError = error; + const guardTimeout = isIndexLockGuardTimeout(error); + cliError( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + (guardTimeout + ? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.' + : 'Watch mode is stopping because the live index may have been updated in place.'), + guardTimeout + ? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath } + : undefined, + ); + stopWatching(); + return; + } + const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; + cliWarn( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + `Retry scheduled; last success ${lastSuccess}.`, + ); + }, + (error) => { + fatalRefreshError = error; + cliError( + `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping.', + ); + stopWatching(); + }, + process.env[GITNEXUS_RC_ENV]?.trim() || undefined, + ); + } catch (error) { + cliError( + ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + + await stopped; + await loop.close(); + if (fatalRefreshError !== undefined) process.exitCode = 1; + } finally { + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + } } finally { - process.removeListener('SIGINT', stop); - process.removeListener('SIGTERM', stop); + setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); + setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); } } finally { - setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); - setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); - setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); + process.removeListener('message', onControlMessage); + process.removeListener('disconnect', cancel); } } diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 07351dee5..4c0687dc7 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -15,7 +15,6 @@ import v8 from 'v8'; import cliProgress from 'cli-progress'; import { formatAnalyzeFtsSkipSummary } from '../core/search/fts-policy.js'; import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js'; -import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js'; import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js'; import { causeChain } from '../lib/utils.js'; import { @@ -51,7 +50,19 @@ import { GitNexusRcError, } from './analyze-config.js'; import type { AnalyzeOptions } from './analyze-options.js'; -import { runFullAnalysis } from '../core/run-analyze.js'; +import { + parseCliPreparedAnalysis, + receiveCliAnalysisContinuation, + type CliAnalysisContinuation, + type CliPreparedAnalysisMessage, +} from './analyze-phase-ipc.js'; +import { AnalysisSupervisorRequired } from '../core/analysis-supervision.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; +import type { AnalysisResourceBudget } from '../core/analysis-resource-budget.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; import { @@ -369,6 +380,8 @@ const shouldBridgeRespawnProgressTty = (): boolean => process.stderr.isTTY === true || process.stdout.isTTY === true; interface RespawnExit { + prepared?: CliPreparedAnalysisMessage; + writerPid?: number; status?: number | null; signal?: NodeJS.Signals | null; stdout?: string; @@ -402,33 +415,81 @@ const appendOutputTail = (tail: string, chunk: unknown): string => { const runRespawnedAnalyze = ( args: readonly string[], env: NodeJS.ProcessEnv, + continuation?: CliAnalysisContinuation, ): Promise => new Promise((resolve) => { let stdout = ''; let stderr = ''; let settled = false; let forwardedSignal: NodeJS.Signals | undefined; + let prepared: CliPreparedAnalysisMessage | undefined; + let phaseError: string | undefined; + let cancelTimer: ReturnType | undefined; const child = spawn(process.execPath, [...args], { - stdio: ['inherit', 'pipe', 'pipe'], + stdio: ['inherit', 'pipe', 'pipe', 'ipc'], windowsHide: true, env, }); const forwardSignal = (signal: NodeJS.Signals): void => { forwardedSignal ??= signal; - if (child.exitCode === null && child.signalCode === null) child.kill(signal); + // This is a standalone process, not a native worker thread. A wedged + // process may be killed after grace; dirty/staged receipts recover it. + cancelTimer ??= setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + }, 60_000); + if (child.connected) { + try { + child.send({ type: 'cancel' }, () => { + /* Actual exit remains the release boundary. */ + }); + } catch { + /* A disconnected child cancels at its next safe point. */ + } + } }; const forwardSigint = () => forwardSignal('SIGINT'); const forwardSigterm = () => forwardSignal('SIGTERM'); const finish = (exit: RespawnExit): void => { if (settled) return; settled = true; + clearTimeout(cancelTimer); process.removeListener('SIGINT', forwardSigint); process.removeListener('SIGTERM', forwardSigterm); - resolve({ ...exit, forwardedSignal }); + resolve({ + ...exit, + ...(phaseError ? { status: 1, message: phaseError } : {}), + forwardedSignal, + prepared, + writerPid: child.pid, + }); }; - process.once('SIGINT', forwardSigint); - process.once('SIGTERM', forwardSigterm); + process.on('SIGINT', forwardSigint); + process.on('SIGTERM', forwardSigterm); + + child.on('message', (message: unknown) => { + try { + const type = (message as { type?: string })?.type; + if (type === 'graph-prepared') { + if (continuation || prepared) throw new Error('Unexpected repeated graph phase'); + prepared = parseCliPreparedAnalysis(message); + } else if (type === 'analysis-continuation-ready' && continuation) { + child.send(continuation, (error) => { + if (error) phaseError = error.message; + }); + } else if ( + (type === 'analysis-ready' || type === 'analysis-watch-ready') && + forwardedSignal + ) { + // Cancellation may arrive while the child is still loading modules. + forwardSignal(forwardedSignal); + } + } catch (error) { + // Never kill a native-active child for malformed IPC. Its exit remains + // the release boundary and a failed projection cannot start a model. + phaseError = error instanceof Error ? error.message : String(error); + } + }); child.stdout?.on('data', (chunk) => { stdout = appendOutputTail(stdout, chunk); @@ -544,6 +605,18 @@ const forceHeapOOMForTestIfEnabled = (): void => { // `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value. const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024; +/** CLI and watch must preserve the same explicit heap/autopilot opt-outs. */ +export function usesManualAnalysisHeap(memoryBudget?: string): boolean { + return ( + memoryBudget === undefined && + (process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP === '1' || + (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && + (memoryAutopilotDisabled() || + parseMaxOldSpaceMb(process.execArgv.join(' ')) !== null || + (parseMaxOldSpaceMb(process.env.NODE_OPTIONS ?? '') ?? 0) >= RESPAWN_HEAP_MB))) + ); +} + /** * Last `--max-old-space-size` value (MB) in a NODE_OPTIONS string, or `null` * when absent/unparseable. Last occurrence wins, matching V8's own @@ -681,9 +754,42 @@ export function resolveBudgetHeap(input: BudgetHeapInput): BudgetHeapDecision { * respawned) so the parse phase's remedy text advises by source. Explicit * heap pins and `GITNEXUS_MEMORY=off` intentionally leave it unset. */ export async function ensureHeap( - options: { cleanForwardedTermination?: boolean; memoryBudget?: string } = {}, + options: { + cleanForwardedTermination?: boolean; + memoryBudget?: string; + admittedHeapMb?: number; + } = {}, ): Promise { const nodeOpts = process.env.NODE_OPTIONS || ''; + if ( + options.memoryBudget === undefined && + options.admittedHeapMb !== undefined && + !(Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0) + ) { + const decision = resolveBudgetHeap({ + budgetMb: options.admittedHeapMb, + execArgv: process.execArgv, + nodeOptions: nodeOpts, + autoCapMb: RESPAWN_HEAP_MB, + autopilotDisabled: false, + inheritedSource: 'budget', + }); + if (!decision.respawn) return false; + const ambient = parseMaxOldSpaceMb(nodeOpts); + if (ambient !== null && process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1') { + cliWarn( + ` NODE_OPTIONS pins the heap to ${ambient}MB. Re-running analyze with the admitted ${options.admittedHeapMb}MB main heap.\n`, + ); + } + return respawnWithHeap( + `--max-old-space-size=${decision.sizing.oldSpaceMb}`, + `--max-semi-space-size=${decision.sizing.semiSpaceMb}`, + 'auto', + 'Analysis likely ran out of memory in its admitted main heap. Raise --resource-budget or reduce the analysis scope. If this persists, it may be a native crash unrelated to heap size.', + nodeOpts, + options, + ); + } // The budget is checked BEFORE the GITNEXUS_MEMORY=off return: an explicit // flag is a deliberate per-run choice, which the opt-out does not cover. if (options.memoryBudget !== undefined) { @@ -766,16 +872,16 @@ export async function ensureHeap( /** Run the analyze child with the given heap flags and map its exit onto this process. */ async function respawnWithHeap( - heapFlag: string, - semiFlag: string, - source: HeapLimitSource, + heapFlag: string | undefined, + semiFlag: string | undefined, + source: HeapLimitSource | undefined, oomGuidance: string, nodeOpts: string, options: { cleanForwardedTermination?: boolean }, ): Promise { // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it // only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS. - const cliFlags = [heapFlag, semiFlag]; + const cliFlags = [heapFlag, semiFlag].filter((flag): flag is string => flag !== undefined); if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG); // Preserve the parent's node flags (execArgv) — dropping them breaks any @@ -789,11 +895,28 @@ async function respawnWithHeap( const childArgs = [...preservedExecArgv, ...cliFlags, ...process.argv.slice(1)]; const childEnv = { ...process.env, - NODE_OPTIONS: `${nodeOpts} ${heapFlag} ${semiFlag}`.trim(), + NODE_OPTIONS: [nodeOpts, heapFlag, semiFlag].filter(Boolean).join(' ').trim(), [HEAP_LIMIT_SOURCE_ENV]: source, + GITNEXUS_ANALYSIS_SUPERVISED: '1', + GITNEXUS_ANALYSIS_MANUAL_HEAP: heapFlag === undefined ? '1' : '0', + GITNEXUS_ANALYSIS_SUPERVISOR_BYTES: + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES ?? String(analysisSupervisorReservation()), }; if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1'; - const childExit = await runRespawnedAnalyze(childArgs, childEnv); + let childExit = await runRespawnedAnalyze(childArgs, childEnv); + if ( + childExit.status === 0 && + !childExit.signal && + !childExit.forwardedSignal && + childExit.prepared + ) { + if (!childExit.writerPid) throw new Error('Graph process did not report its PID'); + childExit = await runRespawnedAnalyze( + childArgs, + { ...childEnv, GITNEXUS_ANALYSIS_CONTINUATION: '1' }, + { type: 'continue-analysis', writerPid: childExit.writerPid, prepared: childExit.prepared }, + ); + } if (childExit.forwardedSignal !== undefined) { process.exitCode = forwardedSignalExitCode( childExit.forwardedSignal, @@ -830,6 +953,8 @@ async function respawnWithHeap( * for it in the first place. */ const ANALYZE_CLI_ENV_KEYS = [ + 'GITNEXUS_ANALYSIS_MAIN_HEAP_MB', + 'GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', HEAP_LIMIT_SOURCE_ENV, 'GITNEXUS_VERBOSE', 'GITNEXUS_PROFILE_DEFERRED', @@ -903,19 +1028,12 @@ export const analyzeCommand = async ( // Snapshot before ensureHeap: it records GITNEXUS_HEAP_LIMIT_SOURCE on a // kept process, which must not leak into a later programmatic call. const envSnap = snapshotAnalyzeEnv(); - if (await ensureHeap({ memoryBudget: options?.memoryBudget })) return; - forceHeapOOMForTestIfEnabled(); - // Install fatal handlers immediately after re-exec resolution so any + // Install fatal handlers before admission so any // async error that escapes the try/catch below (#1169) surfaces with // a stack trace and a non-zero exit code instead of a silent exit 0. installFatalHandlers(); - // npm-11 npx-crash nudge (#1939). Runs here, after the heap re-exec guard, - // so it fires once in the working process and never on the lazy-startup path - // of other commands (e.g. `gitnexus mcp`). - warnIfNpm11NpxRisk(); - // Snapshot the GITNEXUS_* env vars that the impl writes for downstream // consumption, so they don't leak across `analyzeCommand` invocations in // programmatic callers (tests, long-running hosts). `process.exit(0)` on @@ -1285,6 +1403,61 @@ const analyzeCommandImpl = async ( process.env.GITNEXUS_EMBEDDING_DIMS = dims; } + let resources: AnalysisResourceBudget; + let manualHeap = false; + try { + const supervisorBytes = + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined + ? analysisSupervisorReservation() + : Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES); + manualHeap = usesManualAnalysisHeap(options.memoryBudget); + resources = resolveHostAnalysisResources({ + totalMb: + options.resourceBudget === undefined + ? undefined + : parseIntegerOption(options.resourceBudget, '--resource-budget', { minimum: 1 }), + mainHeapMb: + options.memoryBudget !== undefined + ? parseMemoryBudgetMb(options.memoryBudget) + : process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1' && + process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB !== undefined + ? Number(process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB) + : manualHeap + ? Math.ceil(v8.getHeapStatistics().heap_size_limit / 1024 ** 2) + : undefined, + workers: workerPoolSize, + includeEmbeddings: await analysisMayEmbed(repoPath, { + embeddings: embeddingsEnabled, + force: options.force, + dropEmbeddings: options.dropEmbeddings, + branch: cliOptions?.branch, + }), + supervisorBytes, + }); + // The two child phases retain the same frontend and heap reservation. + // Re-measuring either in the child would add another supervisor or trigger + // a nested heap respawn with a slightly different allocation. + process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB = String(resources.mainHeapMb); + process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES = String(supervisorBytes); + if ( + !manualHeap && + (await ensureHeap({ + memoryBudget: options.memoryBudget, + admittedHeapMb: resources.mainHeapMb, + })) + ) + return; + forceHeapOOMForTestIfEnabled(); + } catch (error) { + cliError(` ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + return; + } + + // The npm compatibility warning probes subprocesses. Run it only after the + // heap re-exec guard. A continuation's graph process already emitted it. + if (process.env.GITNEXUS_ANALYSIS_CONTINUATION !== '1') warnIfNpm11NpxRisk(); + // Custom-endpoint UX, emitting at most ONE message that reflects THIS run's // intent (not ambient env). Order matters — the first matching branch wins: // 1. flags given but --embeddings absent: the endpoint won't be used, so @@ -1463,29 +1636,38 @@ const analyzeCommandImpl = async ( bar.start(100, 0, { phase: 'Initializing...' }); - // Graceful SIGINT handling. Pino's default destination is `sync: false` - // (buffered) — flush before exit so in-flight records reach stderr. - // See `gitnexus/src/core/logger.ts:flushLoggerSync`. - let aborted = false; - const sigintHandler = () => { - if (aborted) process.exit(1); - aborted = true; - bar.stop(); - console.log('\n Interrupted — cleaning up...'); - // Bounded CHECKPOINT-then-exit (#2264 review P3): skip the native close (the - // LadybugDB destructor can double-free after --pdg writes), but don't hang - // behind a long --pdg COPY holding the connection lock — bound it so a single - // Ctrl-C stays responsive; the WAL replays on the next analyze. A second - // Ctrl-C (`if (aborted) process.exit(1)` above) remains the escape hatch. - void boundedCheckpointBeforeExit({ - exitCode: 130, - beforeExit: async () => { - const { flushLoggerSync } = await import('../core/logger.js'); - flushLoggerSync(); - }, - }); + // Signals and IPC only request cancellation. The analysis promise owns its + // native calls and cleanup; even a repeated interrupt must not exit mid-call. + const cancellation = new AbortController(); + let cancellationExitCode = 130; + const requestCancellation = (code: number) => { + if (cancellation.signal.aborted) return; + cancellationExitCode = code; + cancellation.abort(new Error('Analysis cancelled')); + console.log('\n Interrupted — waiting for native work to finish...'); }; + const sigintHandler = () => requestCancellation(130); + const sigtermHandler = () => requestCancellation(143); + const cancelMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'cancel') requestCancellation(130); + }; + const disconnected = () => requestCancellation(1); process.on('SIGINT', sigintHandler); + process.on('SIGTERM', sigtermHandler); + process.on('message', cancelMessage); + process.on('disconnect', disconnected); + if (process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1') { + if (!process.connected || !process.send) disconnected(); + else { + try { + process.send({ type: 'analysis-ready' }, (error) => { + if (error) disconnected(); + }); + } catch { + disconnected(); + } + } + } // Route console output through bar.log() to prevent progress bar corruption. // This is a deliberate UI pattern (not a logging concern): analyze runs a @@ -1532,14 +1714,23 @@ const analyzeCommandImpl = async ( } }, 1000); - const t0 = Date.now(); - // ── Run shared analysis orchestrator ─────────────────────────────── try { + // The heap supervisor never runs the pipeline. Load it only in the + // admitted working process, after validation and the re-exec guard. + const analysis = await import('../core/run-analyze.js'); + const continuation = + process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' + ? await receiveCliAnalysisContinuation(cancellation.signal) + : undefined; + cancellation.signal.throwIfAborted(); + const t0 = continuation?.prepared.startedAt ?? Date.now(); const skipAll = options.indexOnly; const skipAgentsMd = skipAll || options.skipAgentsMd; const skipSkills = skipAll || options.skipSkills; const runOptions = { + graphPublication: 'auto' as const, + resources, // The user's own --force only. --skills (skill generation needs a fresh // pipelineResult) and --no-parse-cache (bypassing parser output means // anything only when the pipeline runs) each force the rebuild inside @@ -1604,6 +1795,11 @@ const analyzeCommandImpl = async ( skipNativeCloseOnExit: true, }; const runCallbacks = { + signal: cancellation.signal, + onGraphWork: () => { + if (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' || !process.connected) + throw new AnalysisSupervisorRequired(); + }, onProgress: (_phase, percent, message) => { updateBar(percent, message); }, @@ -1617,11 +1813,45 @@ const analyzeCommandImpl = async ( // further down) writes to them, so selfCommitContextFiles can tell a // pre-existing unstaged user edit apart from this run's stats refresh // and refuse to sweep the former into the latter's commit. - const selfCommitSafety = - options.selfCommit === true + const selfCommitSafety = continuation?.prepared.selfCommitSafety + ? new Map([ + ['AGENTS.md', continuation.prepared.selfCommitSafety.agentsMd], + ['CLAUDE.md', continuation.prepared.selfCommitSafety.claudeMd], + ]) + : options.selfCommit === true ? snapshotSelfCommitSafety(repoPath, ['AGENTS.md', 'CLAUDE.md']) : undefined; - const result = await runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); + const result = continuation + ? { + ...continuation.prepared.result, + stats: + ( + await analysis.continuePreparedAnalysis( + continuation.prepared.handoff, + continuation.writerPid, + runOptions, + runCallbacks, + cancellation.signal, + ) + ).stats ?? {}, + rebuildReasons: [], + pipelineResult: undefined, + preparedGraph: undefined, + alreadyUpToDate: false, + ftsRepairedOnly: false, + graphWriteCollapsed: undefined, + } + : await analysis.runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); + cancellation.signal.throwIfAborted(); + + let preparedMessage: CliPreparedAnalysisMessage | undefined; + if (result.preparedGraph) { + const { projectCliPreparedAnalysis, requireCliAnalysisSupervisor } = + await import('./analyze-phase-ipc.js'); + requireCliAnalysisSupervisor(); + // Validate identity before skill/context writes use the registered name. + preparedMessage = projectCliPreparedAnalysis(result, repoPath, t0, selfCommitSafety); + } if (result.alreadyUpToDate) { // Even the fast path must prove the repo is discoverable. A prior @@ -1701,7 +1931,7 @@ const analyzeCommandImpl = async ( // success so the silent-finalize state surfaces with a non-zero // exit code and an actionable error instead of being mistaken for // a healthy index. - await assertAnalysisFinalized(repoPath, result.storagePath); + if (!preparedMessage) await assertAnalysisFinalized(repoPath, result.storagePath); // Skill generation (CLI-only, uses pipeline result from analysis). // Gated so `--index-only --skills` skips community skill writes too @@ -1765,6 +1995,24 @@ const analyzeCommandImpl = async ( } } + if (preparedMessage) { + // Skill generation above needs the live graph and stays in this child. + // Publication, embeddings, the success banner and self-commit belong to + // the supervisor only after this process has actually exited. + const { sendCliPreparedAnalysis } = await import('./analyze-phase-ipc.js'); + await sendCliPreparedAnalysis(preparedMessage); + clearInterval(elapsedTimer); + process.removeListener('SIGINT', sigintHandler); + console.log = origLog; + // eslint-disable-next-line no-console -- restoring progress-bar routing + console.warn = origWarn; + // eslint-disable-next-line no-console -- restoring progress-bar routing + console.error = origError; + bar.stop(); + process.exitCode = 0; + return; + } + // #2639: opt-in self-commit of any AGENTS.md/CLAUDE.md churn written by // this run (the primary generateAIContextFiles call inside // runFullAnalysis, and/or the --skills regeneration above). Best-effort @@ -1842,6 +2090,31 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); + if (cancellation.signal.aborted) { + process.exitCode = cancellationExitCode; + return; + } + + if (err instanceof AnalysisSupervisorRequired) { + const sizing = resolveBudgetHeap({ + budgetMb: resources.mainHeapMb, + execArgv: process.execArgv, + nodeOptions: process.env.NODE_OPTIONS ?? '', + autoCapMb: RESPAWN_HEAP_MB, + autopilotDisabled: false, + inheritedSource: 'budget', + }).sizing; + await respawnWithHeap( + manualHeap ? undefined : `--max-old-space-size=${sizing.oldSpaceMb}`, + manualHeap ? undefined : `--max-semi-space-size=${sizing.semiSpaceMb}`, + manualHeap ? (process.env[HEAP_LIMIT_SOURCE_ENV] as HeapLimitSource | undefined) : 'auto', + 'Analysis exceeded its resource budget.', + process.env.NODE_OPTIONS ?? '', + {}, + ); + return; + } + const msg = err instanceof Error ? err.message : String(err); // Registry name-collision from --name (#829) — surface as an @@ -2195,6 +2468,12 @@ const analyzeCommandImpl = async ( process.exitCode = 1; return; + } finally { + clearInterval(elapsedTimer); + process.removeListener('SIGINT', sigintHandler); + process.removeListener('SIGTERM', sigtermHandler); + process.removeListener('message', cancelMessage); + process.removeListener('disconnect', disconnected); } // LadybugDB's native module holds open handles that prevent Node from exiting. diff --git a/gitnexus/src/cli/clean.ts b/gitnexus/src/cli/clean.ts index 1ac98cdf4..2913b6e14 100644 --- a/gitnexus/src/cli/clean.ts +++ b/gitnexus/src/cli/clean.ts @@ -34,6 +34,7 @@ import { } from '../core/lbug/sidecar-recovery.js'; import { t } from './i18n/index.js'; import { getGlobalDir } from '../storage/global-dir.js'; +import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js'; import { STORES_DIR } from '../storage/shared-store.js'; import { findLegacyLocalIndex, @@ -271,8 +272,8 @@ export const cleanCommand = async (options?: { console.log(t('clean.branchNotIndexed', { branch: options.branch })); return; } - const { lbugPath } = getStoragePaths(repo.repoPath, summary.branch, storagePath); - const branchDir = path.dirname(lbugPath); + // Cleanup targets the owned slot, even when its graph pointer is damaged. + const branchDir = path.join(storagePath, BRANCHES_DIR, branchSlug(summary.branch)); if (!isContainedBranchDir(storagePath, branchDir)) { logger.error( `Refusing to clean branch index outside the validated storage slot: ${branchDir}`, diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts index b4b6626fb..bbf544057 100644 --- a/gitnexus/src/cli/embeddings-sync.ts +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -1,36 +1,16 @@ import { lstat } from 'node:fs/promises'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; +import { resolveGraphPath } from '../storage/shared-store.js'; +import { resolveHostAnalysisResources } from '../core/analysis-host-resources.js'; +import { readPreparedAnalysis } from '../core/analysis-publication.js'; import path from 'node:path'; import { cliInfo } from './cli-message.js'; import { getGitRoot } from '../storage/git.js'; -import { - acquireIndexLock, - requireExclusiveIndexLock, - sweepStagingArtifacts, -} from '../storage/index-lock.js'; -import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'; -import { - closeLbug, - executeQuery, - executeWithReusedStatement, - fetchExistingEmbeddingHashes, - initLbug, -} from '../core/lbug/lbug-adapter.js'; -import { - decideEmbeddingResume, - mintInterruptedCheckpoint, - mintPartialCheckpoint, - mintUnverifiedCountCheckpoint, - type EmbeddingCheckpoint, - type EmbeddingCheckpointProgress, -} from '../core/embedding-checkpoint.js'; +import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; +import { getStoragePaths, loadMeta } from '../storage/repo-manager.js'; +import { decideEmbeddingResume } from '../core/embedding-checkpoint.js'; import { EMBEDDING_DIMS, embeddingDimsMismatch } from '../core/lbug/schema.js'; -import type { RepoMeta } from '../storage/repo-meta.js'; -import { - measurePersistedEmbeddingCount, - persistedEmbeddingCountOrUndefined, -} from '../core/embedding-count.js'; import { isHttpMode } from '../core/embeddings/http-client.js'; import { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS, @@ -42,8 +22,6 @@ import { assessLocalEmbeddingRuntime, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js'; -import { reapEmbeddingSidecarSafely } from '../core/embeddings/embedding-sidecar-reap.js'; -import { contentRetentionFromMeta } from '../core/content-retention.js'; /** Add missing embeddings directly to a healthy index, checkpointing periodically. */ export const embeddingsSyncCommand = async (inputPath?: string): Promise => { @@ -54,8 +32,11 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = const metaDir = path.dirname(metaPath); // Writes go to the slot's own graph. A shared-store checkout that reads an // immutable commit graph (#3352) takes a private copy first. - const lbugPath = path.join(metaDir, LBUG_DIRECTORY); - const lock = await acquireIndexLock(metaDir, { sweep: false }); + const resources = resolveHostAnalysisResources({ + includeParsing: false, + includeEmbeddings: true, + }); + const lock = await acquireAnalysisSlotLock(metaDir, { sweep: false }); try { requireExclusiveIndexLock( lock, @@ -70,6 +51,8 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = 'Run `gitnexus analyze` to recover them first.', ); } + if (await readPreparedAnalysis(metaDir)) + throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.'); sweepStagingArtifacts(metaDir); if (!(await ensurePrivateSharedGraph(metaDir, (m) => console.log(` ${m}`)))) { throw new Error('The shared graph this checkout reads is gone. Run gitnexus analyze first.'); @@ -81,6 +64,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = throw new Error('The structural index is incomplete. Run gitnexus analyze --force first.'); } + const lbugPath = resolveGraphPath(metaDir); let lbugStat; try { lbugStat = await lstat(lbugPath); @@ -97,8 +81,6 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js'); const identity = resolveEmbeddingIdentity(); - let forceReembedNodeIds: ReadonlySet | undefined; - let resumedFrom: EmbeddingCheckpoint | undefined; if (meta.embeddingCheckpoint) { const checkpoint = meta.embeddingCheckpoint; const decision = decideEmbeddingResume(checkpoint, identity); @@ -125,25 +107,15 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = ); } cliInfo(decision.log); - if (decision.action === 'resume') { - forceReembedNodeIds = decision.pendingNodeIds; - resumedFrom = decision.resumedFrom; - } } - // The vector column is FLOAT[N] fixed when the index was built, and the - // pipeline deletes each batch's stale rows immediately before inserting the - // replacements — so a width change here deletes rows it cannot re-insert. - // `analyze` forces a full rebuild on the same mismatch; only a rebuild can - // retype the column, so this writer refuses instead. An absent recorded - // width is not a mismatch (see `embeddingDimsMismatch`). - if (embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) { + // Keep legacy sync's explicit migration requirement; the new vector store + // can use a different width once analyze has established its own generation. + if (!meta.graphGeneration && embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) throw new Error( `Cannot sync embeddings: this index stores FLOAT[${meta.embeddingDims}] vectors, ` + - `but this run embeds at ${EMBEDDING_DIMS} dimensions. ` + - 'Run `gitnexus analyze --embeddings --force` to rebuild the column at the new width.', + `but this run embeds at ${EMBEDDING_DIMS} dimensions. Run gitnexus analyze --embeddings --force.`, ); - } if (!isHttpMode()) { const assessment = assessLocalEmbeddingRuntime(); @@ -167,93 +139,27 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = } } - await initLbug(lbugPath); - try { - const existing = await fetchExistingEmbeddingHashes(executeQuery); - let lastPercent = -1; - - const countEmbeddings = async (): Promise => - persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery)); - // One write path for every meta update this command makes. The re-read - // happens immediately before each save so a concurrent writer's fields - // survive. #2790 traced two production drifts to hand-copied writers of - // these exact fields, so this file keeps one copy instead of three. - const persistMeta = async (patch: (latest: RepoMeta) => Partial): Promise => { - const latest = (await loadMeta(metaDir)) ?? meta; - await saveMeta(metaDir, { ...latest, ...patch(latest) }); - }; - const saveCheckpoint = async ( - checkpoint: EmbeddingCheckpointProgress, - pendingNodeIds: string[], - embeddings?: number, - ): Promise => - persistMeta((latest) => ({ - ...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }), - embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds), - })); - - cliInfo(`Embedding ${repoPath}`); - cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`); - - const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js'); - const result = await runEmbeddingPipeline( - executeQuery, - executeWithReusedStatement, - (progress) => { - const percent = Math.floor(progress.percent); - if (percent !== lastPercent && (percent % 5 === 0 || percent === 100)) { - lastPercent = percent; - cliInfo( - ` ${percent}% — ${progress.nodesProcessed ?? 0}/${progress.totalNodes ?? '?'} nodes`, - ); + cliInfo(`Embedding ${repoPath}`); + let lastPercent = -1; + const { runAnalysisEmbeddingPhase } = await import('../core/analysis-embedding-phase.js'); + const completed = await runAnalysisEmbeddingPhase({ + metadataDirectory: metaDir, + graphPath: lbugPath, + resources, + nodeLimit: 0, + expectedGeneration: meta.graphGeneration?.id, + callbacks: { + onLog: cliInfo, + onProgress: (_phase, percent, message) => { + const rounded = Math.floor(percent); + if (rounded !== lastPercent) { + lastPercent = rounded; + cliInfo(` ${rounded}% — ${message}`); } }, - {}, - undefined, - existing && existing.size ? existing : undefined, - { - source: { - repoPath, - fileHashes: meta.fileHashes, - contentRetention: contentRetentionFromMeta(meta), - }, - forceReembedNodeIds, - onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { - await saveCheckpoint(checkpoint, nodeIds); - }, - onCheckpoint: async (checkpoint) => { - await saveCheckpoint(checkpoint, [], await countEmbeddings()); - }, - }, - ); - - const embeddings = await countEmbeddings(); - if (embeddings === undefined) { - // Keep last-known stats.embeddings. An interrupted window marker would - // fail the identity gate on the next run even though this run finished; - // unverified-count is the recovery kind that forces a recount (#2790). - await persistMeta(() => ({ - embeddingCheckpoint: result.failedNodeIds.length - ? mintPartialCheckpoint(identity, result, resumedFrom) - : mintUnverifiedCountCheckpoint(identity, { - nodesProcessed: result.nodesProcessed, - totalNodes: result.nodesProcessed, - chunksProcessed: result.chunksProcessed, - }), - })); - throw new Error('Could not verify persisted embedding count.'); - } - await persistMeta((latest) => ({ - stats: { ...latest.stats, embeddings }, - embeddingCheckpoint: result.failedNodeIds.length - ? mintPartialCheckpoint(identity, result, resumedFrom) - : undefined, - })); - cliInfo(`Embeddings ready: ${embeddings}`); - } finally { - await closeLbug().catch(() => {}); - await reapEmbeddingSidecarSafely(); - } + }, + }); + cliInfo(`Embeddings ready: ${completed.stats?.embeddings ?? 0}`); } finally { lock.release(); } diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index 7a3ff451f..85474843d 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -74,6 +74,7 @@ const OPTION_DESCRIPTION_KEYS = { 'analyze|--worker-timeout ': 'help.option.analyze.workerTimeout', 'analyze|--wal-checkpoint-threshold ': 'help.option.analyze.walCheckpointThreshold', 'analyze|--memory-budget ': 'help.option.analyze.memoryBudget', + 'analyze|--resource-budget ': 'help.option.analyze.resourceBudget', 'analyze|--workers ': 'help.option.analyze.workers', 'analyze|--max-processes ': 'help.option.analyze.maxProcesses', 'analyze|--max-process-branching ': 'help.option.analyze.maxProcessBranching', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 58105eccc..8f041070d 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -311,6 +311,8 @@ export const en = { 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).', 'help.option.analyze.memoryBudget': 'Main-thread V8 heap size in MB for analyze (integer >= 200). Re-runs analyze with exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size pin; parse workers keep their own heap caps.', + 'help.option.analyze.resourceBudget': + 'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.', 'help.option.analyze.workers': 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', 'help.option.analyze.maxProcesses': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index f06374781..f07717d4f 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -284,6 +284,8 @@ export const zhCN = { 'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值（字节，整数 >= -1；默认：67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。', 'help.option.analyze.memoryBudget': 'analyze 主线程 V8 堆大小（MB，整数 >= 200）。以该堆大小重新运行 analyze，覆盖按 RAM/cgroup 自动计算的上限及任何 --max-old-space-size 设置；解析 worker 各自保留独立的堆上限。', + 'help.option.analyze.resourceBudget': + '分析进程的资源总预算（MiB），包括原生数据库缓冲池、解析 worker、嵌入模型和监督进程。', 'help.option.analyze.workers': '解析 worker 池大小（>=1）。默认：cores-1，最多 16，按仓库规模自适应。', 'help.option.analyze.maxProcesses': diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index d89641332..1d750fc9a 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -17,6 +17,7 @@ import { localizeCliHelp } from './help-i18n.js'; import { t } from './i18n/index.js'; import { writeCommandBanner } from './command-banner.js'; import { runProcessCliUpdateNotice } from './update-notice.js'; +import { memoryAutopilotDisabled } from '../core/ingestion/utils/effective-ram.js'; const program = new Command(); @@ -178,6 +179,10 @@ program 'exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size ' + 'pin; parse workers keep their own heap caps.', ) + .option( + '--resource-budget ', + 'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.', + ) .option( '--workers ', 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', @@ -286,14 +291,48 @@ program process.env.GITNEXUS_EMBEDDING_DIMS = dimsEnvBaseline; } }) - .action( - createAnalyzerLbugLazyAction( + .action(async (inputPath, options) => { + // The connected continuation only publishes the writer's prepared graph + // and embeds it. Its analyzer receipt is never used: the graph writer has + // already finalized provenance. Keep the native check, without repeating + // the complete analyzer identity snapshot in this separate process. + // An explicit heap budget without the budget-child marker always re-execs + // in ensureHeap before runFullAnalysis. Its parent receipt is unused too; + // the child still captures provenance before importing the analyzer. + const budgetSupervisor = + !options.watch && + options.memoryBudget !== undefined && + process.env.GITNEXUS_HEAP_LIMIT_SOURCE !== 'budget'; + // An unconfigured automatic-heap parent also always re-execs with the + // admitted old/semi-space flags. Keep conservative capture for custom Node + // flags, manual heaps, test overrides and every analyzing child. + const autoSupervisor = + !options.watch && + options.memoryBudget === undefined && + process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' && + process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP !== '1' && + !memoryAutopilotDisabled() && + process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB === undefined && + process.execArgv.length === 0 && + !(process.env.NODE_OPTIONS ?? '').trim(); + if ( + budgetSupervisor || + autoSupervisor || + (process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected) + ) { + await createLbugLazyAction(() => import('./analyze.js'), 'analyzeCommand')( + inputPath, + options, + ); + return; + } + await createAnalyzerLbugLazyAction( () => import('../core/analyzer-identity.js'), () => import('./analyze.js'), 'analyzeOrWatchCommandWithRunnerIdentity', import.meta.url, - ), - ); + )(inputPath, options); + }); program .command('index [path...]') diff --git a/gitnexus/src/cli/resolve-invocation.ts b/gitnexus/src/cli/resolve-invocation.ts index a371985c5..145b3a98a 100644 --- a/gitnexus/src/cli/resolve-invocation.ts +++ b/gitnexus/src/cli/resolve-invocation.ts @@ -22,6 +22,7 @@ interface InvocationResolver { // no argument. resolveInvocationMode: ( probe?: (command: string, gitnexusWrapper?: boolean) => string | null, + deps?: { readonly npmMajor?: number | null }, ) => InvocationMode; formatDocumentationDlxCommand: ( gitnexusArgs: string, @@ -91,8 +92,17 @@ export function getNpmMajorVersion(): number | null { * cannot run. */ export function warnIfNpm11NpxRisk(): void { - if (resolveInvocationMode() !== 'npx') return; - const major = getNpmMajorVersion(); + // The resolver and warning need the same version. Probe once, lazily: a + // preferred global binary or forced alternative still needs no npm spawn. + let cachedMajor: number | null | undefined; + const deps = { + get npmMajor(): number | null { + if (cachedMajor === undefined) cachedMajor = getNpmMajorVersion(); + return cachedMajor; + }, + }; + if (resolveInvocationMode(undefined, deps) !== 'npx') return; + const major = deps.npmMajor; if (major === null || major < 11) return; process.stderr.write( `Warning: npm ${major}.x can crash while installing gitnexus via npx ` + diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index 8316fc3ae..020b7dd96 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -39,7 +39,9 @@ import { analyzerRunnerIdentitiesEqual, resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js'; -import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { getFtsDisabledReason, FTS_DISABLED_MESSAGE } from '../core/search/fts-policy.js'; import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js'; import { @@ -218,6 +220,9 @@ export const statusCommand = async (options: StatusOptions = {}) => { runnerIdentity: meta.runnerIdentity ?? null, runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown', incompleteReasons, + graphIncompleteReasons: getGraphIncompleteReasons(meta), + embeddingProgress: await readEmbeddingJobProgress(storagePath, meta), + capabilities: await readIndexCapabilities(storagePath, meta), contentRetention: meta.contentRetention ?? 'full', }, current: sourceAvailable ? { commit: getCurrentCommit(entry.path) } : null, @@ -233,6 +238,14 @@ export const statusCommand = async (options: StatusOptions = {}) => { console.log(`Index storage: ${storagePath}`); console.log(`Indexed: ${new Date(meta.indexedAt).toLocaleString()}`); console.log(`Indexed commit: ${meta.lastCommit?.slice(0, 7)}`); + if (incompleteReasons.length) { + console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); + console.log( + `Graph incomplete reasons: ${JSON.stringify(payload.index.graphIncompleteReasons)}`, + ); + } + if (payload.index.embeddingProgress) + console.log(`Embedding progress: ${JSON.stringify(payload.index.embeddingProgress)}`); console.log( sourceAvailable ? 'Status: registered index (use status without --repo for working-tree freshness)' @@ -310,12 +323,15 @@ export const statusCommand = async (options: StatusOptions = {}) => { // commit comparison below then says whether it needs a re-analyze. Legacy/ // no-branch metas and detached HEAD also fall through to the flat index. let activeMeta = repo.meta; + let activeMetaDirectory = repo.storagePath; let workspaceLagsBranch = false; if (currentBranch && repo.meta.branch && currentBranch !== repo.meta.branch) { const { metaPath } = getStoragePaths(repo.repoPath, currentBranch, repo.storagePath); const branchMeta = await loadMeta(path.dirname(metaPath)); - if (branchMeta) activeMeta = branchMeta; - else workspaceLagsBranch = true; + if (branchMeta) { + activeMeta = branchMeta; + activeMetaDirectory = path.dirname(metaPath); + } else workspaceLagsBranch = true; } const currentRunnerIdentity = resolveAnalyzerRunnerIdentity(import.meta.url); @@ -324,10 +340,11 @@ export const statusCommand = async (options: StatusOptions = {}) => { currentRunnerIdentity, ); const incompleteReasons = getIndexIncompleteReasons(activeMeta); + const graphIncompleteReasons = getGraphIncompleteReasons(activeMeta); const metadataIsCurrent = currentCommit === activeMeta.lastCommit && runnerIdentityIsCurrent && - incompleteReasons.length === 0; + graphIncompleteReasons.length === 0; // A matching HEAD is not enough: `analyze` re-indexes changed content at the // same commit, so the files the index covers must still be compared against @@ -353,7 +370,10 @@ export const statusCommand = async (options: StatusOptions = {}) => { contentDrift.reason === 'no-file-hashes' && !isWorkingTreeDirty(repo.repoPath)); - const isUpToDate = metadataIsCurrent && contentIsCurrent; + const graphIsCurrent = metadataIsCurrent && contentIsCurrent; + const isUpToDate = graphIsCurrent && incompleteReasons.length === 0; + const embeddingProgress = await readEmbeddingJobProgress(activeMetaDirectory, activeMeta); + const capabilities = await readIndexCapabilities(activeMetaDirectory, activeMeta); // Shared sibling store (#3352): which graph this checkout reads, and any // pre-adoption index still sitting in /.gitnexus. const storeRoot = storeRootOfCheckoutSlot(repo.storagePath); @@ -386,10 +406,13 @@ export const statusCommand = async (options: StatusOptions = {}) => { index: { indexedAt: activeMeta.indexedAt, commit: activeMeta.lastCommit, - ...(activeMeta.capabilities ? { capabilities: activeMeta.capabilities } : {}), + ...(capabilities ? { capabilities } : {}), runnerIdentity: activeMeta.runnerIdentity ?? null, runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown', incompleteReasons, + graphIncompleteReasons, + graphStatus: graphIsCurrent ? 'up-to-date' : 'stale', + embeddingProgress, }, current: { commit: currentCommit, @@ -452,6 +475,10 @@ export const statusCommand = async (options: StatusOptions = {}) => { ); if (incompleteReasons.length > 0) { console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); + console.log(`Graph status: ${graphIsCurrent ? 'up-to-date' : 'stale'}`); + } + if (embeddingProgress) { + console.log(`Embedding progress: ${JSON.stringify(embeddingProgress)}`); } console.log(`${t('status.currentRunnerIdentity')}: ${JSON.stringify(currentRunnerIdentity)}`); if (contentDrift?.kind === 'current') { diff --git a/gitnexus/src/core/analysis-embedding-phase.ts b/gitnexus/src/core/analysis-embedding-phase.ts new file mode 100644 index 000000000..6dfe67396 --- /dev/null +++ b/gitnexus/src/core/analysis-embedding-phase.ts @@ -0,0 +1,266 @@ +/** Common bounded embedding phase. Callers retain the ordinary slot writer lock. */ +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import lbug from '@ladybugdb/core'; +import type { AnalyzeCallbacks } from './run-analyze.js'; +import type { AnalysisResourceBudget } from './analysis-resource-budget.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { loadMeta, saveMeta } from '../storage/repo-manager.js'; +import { SCHEMA_FINGERPRINT } from './lbug/schema.js'; +import { openLbugConnection, closeLbugConnection } from './lbug/lbug-config.js'; +import { drainAnalysisNativeResources } from './lbug/lbug-adapter.js'; +import { assertLbugPublicationReady } from './lbug/publication.js'; +import { exportLegacyVectors } from './lbug/legacy-vector-export.js'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from './analysis-publication.js'; +import { verifyPublishedAnalysisGraph } from './analysis-graph-verification.js'; +import { runPublishedEmbeddingJob } from './embeddings/embedding-job.js'; +import { drainEmbeddingSidecarForAnalysis } from './embeddings/embedding-sidecar-client.js'; +import { deriveEmbeddingCap } from './embedding-mode.js'; +import { embeddingProgressFromState, type EmbeddingJobProgress } from './analysis-progress.js'; +import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from './analyzer-identity.js'; +import { resolveGraphPath } from '../storage/shared-store.js'; + +export function canPromoteLegacyGraph(metadata: RepoMeta): boolean { + return ( + !metadata.graphGeneration && + !metadata.incrementalInProgress && + !metadata.graphWriteCollapsed && + !metadata.embeddingCheckpoint?.recovery && + metadata.schemaFingerprint === SCHEMA_FINGERPRINT && + metadata.runnerIdentity?.schemaVersion === ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION && + metadata.parseCacheIdentity !== undefined && + metadata.fileHashes !== undefined + ); +} + +/** Adopt a healthy legacy graph without parsing, rewriting or copying its graph. + * Only its vector table is exported, retaining historical provenance for the + * complete-group validator. Interrupted staged graphs take analyze's existing + * recovery path instead. */ +export async function promoteLegacyGraph( + directory: string, + graphPath: string, + metadata: RepoMeta, + resources: AnalysisResourceBudget, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!canPromoteLegacyGraph(metadata)) + throw new Error( + 'Legacy graph provenance requires analysis before embedding; run gitnexus analyze first.', + ); + if ( + path.dirname(graphPath) !== path.resolve(directory) || + path.basename(graphPath) !== (metadata.graphPath ?? 'lbug') + ) + throw new Error('Legacy graph promotion requires an owned private graph'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + signal?.throwIfAborted(); + await assertLbugPublicationReady(graphPath); + const handle = await openLbugConnection(lbug, graphPath, { + readOnly: true, + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }); + let legacy = metadata.legacyVectorSource; + try { + if (!legacy) { + const historical = metadata.embeddingCheckpoint; + legacy = await exportLegacyVectors({ + metadataDirectory: directory, + query: async (sql) => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected legacy migration result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }, + provenance: { + schemaFingerprint: metadata.schemaFingerprint!, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: historical?.pendingNodeIds ?? [], + bufferPoolBytes: resources.embedding.vectorPoolBytes, + threads: resources.embedding.nativeThreads, + signal, + }); + } + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graphPath }); + } + signal?.throwIfAborted(); + // Persist the independent paid source before retiring the monolithic name. + const retained = { ...metadata, ...(legacy ? { legacyVectorSource: legacy } : {}) }; + // Canonical legacy paths are represented by absence in the publication + // contract; shared-store privatization can explicitly record that same name. + if (retained.graphPath === 'lbug') delete retained.graphPath; + await saveMeta(directory, retained); + const now = Date.now(); + const dirty = await beginInPlaceAnalysis( + directory, + { + ...retained, + incrementalInProgress: { + startedAt: now, + updatedAt: now, + phase: 'legacy-migration', + toWriteCount: 0, + }, + }, + (meta) => saveMeta(directory, meta), + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Legacy migration has no owned publication attempt'); + const content = createHash('sha256'); + for (const entry of Object.entries(metadata.fileHashes!).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + )) + content.update(JSON.stringify(entry)).update('\0'); + await prepareAnalysisPublication(directory, graphPath, retained, content.digest('hex'), { + mode: 'in-place', + attemptId, + }); + const published = await publishPreparedAnalysis(directory, { + saveMetadata: (meta) => saveMeta(directory, meta), + verify: (file, meta) => + verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }), + }); + if (!published) throw new Error('Legacy graph publication disappeared'); + return published; +} + +export interface AnalysisEmbeddingPhaseOptions { + metadataDirectory: string; + graphPath: string; + resources: AnalysisResourceBudget; + callbacks: AnalyzeCallbacks; + nodeLimit?: number; + signal?: AbortSignal; + /** Reject a superseded handoff before any model or paid provider request. */ + expectedGeneration?: string; +} + +export async function runAnalysisEmbeddingPhase( + options: AnalysisEmbeddingPhaseOptions, +): Promise { + const { metadataDirectory, resources, callbacks, signal } = options; + signal?.throwIfAborted(); + let metadata = await loadMeta(metadataDirectory); + if (await readPreparedAnalysis(metadataDirectory)) + throw new Error('A prepared graph awaits publication; run gitnexus analyze before embedding.'); + if (!metadata || metadata.incrementalInProgress || metadata.graphWriteCollapsed) + throw new Error('The graph is incomplete; run gitnexus analyze before embedding.'); + if (!Number.isSafeInteger(metadata.stats?.nodes) || (metadata.stats?.nodes ?? -1) < 0) + throw new Error('The graph node count is unverified; run gitnexus analyze before embedding.'); + if (options.expectedGeneration && metadata.graphGeneration?.id !== options.expectedGeneration) + throw new Error('Graph generation changed before embedding; retry analysis.'); + const cap = deriveEmbeddingCap( + metadata.stats?.nodes ?? 0, + metadata.embeddingJob || metadata.embeddingCheckpoint ? 0 : options.nodeLimit, + ); + if (cap.skipForCap) { + callbacks.onLog?.( + `Embeddings skipped: graph exceeds the ${cap.nodeLimit}-node cap; use --embeddings 0 to disable it.`, + ); + await callbacks.onGraphPublished?.(); + return metadata; + } + const { isHttpMode } = await import('./embeddings/http-client.js'); + if (!isHttpMode() && resources.embedding.modelBytes === 0) + throw new Error( + 'Local embedding work was not admitted in this resource budget; retry analysis.', + ); + if (metadata.embeddingCheckpoint) { + const { decideEmbeddingResume } = await import('./embedding-checkpoint.js'); + const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js'); + const identity = resolveEmbeddingIdentity(); + const checkpoint = metadata.embeddingCheckpoint; + if (Object.hasOwn(checkpoint, 'recovery')) + throw new Error('The checkpoint references staged embeddings; run gitnexus analyze first.'); + const decision = decideEmbeddingResume(checkpoint, identity); + if (decision.action === 'abort') throw new Error(decision.error); + if ( + checkpoint.provider !== identity.provider || + checkpoint.model !== identity.model || + checkpoint.dimensions !== identity.dimensions + ) + throw new Error( + 'The embedding checkpoint has a different provider, model or width; run gitnexus analyze --embeddings --force.', + ); + callbacks.onLog?.(decision.log); + } + if (!metadata.graphGeneration) + metadata = await promoteLegacyGraph( + metadataDirectory, + options.graphPath, + metadata, + resources, + signal, + ); + const graphPath = resolveGraphPath(metadataDirectory, metadata); + let progress: EmbeddingJobProgress = { phase: 'preparing' }; + callbacks.onProgress('graph-ready', 90, 'Graph available; preparing embeddings', { + graphPhase: 'ready', + embeddingProgress: progress, + }); + try { + await callbacks.onGraphPublished?.(); + const result = await runPublishedEmbeddingJob({ + metadataDirectory, + graphPath, + resources: resources.embedding, + signal, + config: { threads: resources.embedding.nativeThreads }, + log: callbacks.onLog, + metadata: { + loadMetadata: () => loadMeta(metadataDirectory), + saveMetadata: (meta) => saveMeta(metadataDirectory, meta), + }, + onProgress: (state) => { + progress = embeddingProgressFromState(state); + const completed = state.completedNodes + state.reusedNodes; + const percent = + state.phase === 'indexing' ? 99 : 90 + 8 * (completed / Math.max(1, state.totalNodes)); + callbacks.onProgress( + 'embeddings', + percent, + state.phase === 'indexing' + ? 'Indexing completed embeddings' + : `Embedding ${completed}/${state.totalNodes}`, + { graphPhase: 'ready', embeddingProgress: progress }, + ); + }, + }); + if (result.phase !== 'ready') + throw new Error( + 'Embedding work remains incomplete; the graph is available. Retry to resume durable progress.', + ); + return result.metadata; + } catch (error) { + callbacks.onProgress('embeddings', 98, 'Embeddings incomplete; graph remains available', { + graphPhase: 'ready', + embeddingProgress: { ...progress, phase: 'failed' }, + }); + throw error; + } +} diff --git a/gitnexus/src/core/analysis-graph-verification.ts b/gitnexus/src/core/analysis-graph-verification.ts new file mode 100644 index 000000000..9999e6578 --- /dev/null +++ b/gitnexus/src/core/analysis-graph-verification.ts @@ -0,0 +1,83 @@ +/** Read-only verification after the graph writer has physically released its DB. */ +import lbug from '@ladybugdb/core'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_FINGERPRINT } from './lbug/schema.js'; +import { closeLbugConnection, openLbugConnection, estimateBufferPool } from './lbug/lbug-config.js'; +import { assertLbugPublicationReady } from './lbug/publication.js'; +import { getFtsIndexes } from './search/fts-schema.js'; +import { verifySearchFTSIndexes } from './search/fts-indexes.js'; +import { ExtensionManager } from './lbug/extension-loader.js'; + +function count(value: unknown): number { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Published graph count is unverified'); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) + throw new Error('Published graph count is invalid'); + return result; +} + +export async function verifyPublishedAnalysisGraph( + graphPath: string, + metadata: RepoMeta, + resources: { bufferPoolBytes: number; threads: number }, +): Promise { + if ( + metadata.incrementalInProgress || + metadata.graphWriteCollapsed || + metadata.schemaFingerprint !== SCHEMA_FINGERPRINT || + !metadata.graphGeneration + ) + throw new Error('Prepared graph has no compatible, verified schema generation'); + const expectedNodes = count(metadata.stats?.nodes); + const expectedEdges = count(metadata.stats?.edges); + await assertLbugPublicationReady(graphPath); + const handle = await openLbugConnection(lbug, graphPath, { + ...resources, + bufferPoolBytes: Math.min( + resources.bufferPoolBytes, + estimateBufferPool(expectedNodes + expectedEdges), + ), + readOnly: true, + }); + const query = async (sql: string): Promise[]> => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected publication verification result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }; + try { + // Binding every structural table also rejects incomplete native catalogs. + // Count metadata is bounded; no source text, graph rows or vectors enter JS. + const nodes = await query( + NODE_TABLES.map((table) => `MATCH (n:\`${table}\`) RETURN count(n) AS count`).join( + ' UNION ALL ', + ), + ); + const edges = await query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS count`); + if ( + nodes.length !== NODE_TABLES.length || + edges.length !== 1 || + nodes.reduce((total, row) => total + count(row.count), 0) !== expectedNodes || + count(edges[0].count) !== expectedEdges + ) + throw new Error('Prepared graph counts differ after writer release; artifacts retained'); + if (metadata.capabilities?.fts.status === 'available') { + if ( + !(await new ExtensionManager().ensure(query, 'fts', 'publication verification', { + policy: 'load-only', + })) + ) + throw new Error('Prepared graph FTS extension could not be loaded; artifacts retained'); + const missing = await verifySearchFTSIndexes(query, getFtsIndexes(metadata.ftsProfile)); + if (missing.length) + throw new Error(`Prepared graph search indexes are incomplete: ${missing.join(', ')}`); + } + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graphPath }); + } +} diff --git a/gitnexus/src/core/analysis-host-resources.ts b/gitnexus/src/core/analysis-host-resources.ts new file mode 100644 index 000000000..e4c74a40f --- /dev/null +++ b/gitnexus/src/core/analysis-host-resources.ts @@ -0,0 +1,89 @@ +/** Host inputs for the pure phase-admission policy. No native/model imports. */ +import os from 'node:os'; +import path from 'node:path'; +import { effectiveRamBytes } from './ingestion/utils/effective-ram.js'; +import { granuleRatio } from './lbug/lbug-config.js'; +import { + resolveAnalysisResourceBudget, + UNBOUNDED_GRAPH_POOL_MESSAGE, + type AnalysisResourceBudget, + type AnalysisResourceRequest, +} from './analysis-resource-budget.js'; + +const MIB = 1024 * 1024; + +function positiveEnvironment(name: string): number | undefined { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === '') return undefined; + const value = Number(raw); + if (name === 'GITNEXUS_LBUG_BUFFER_POOL_SIZE' && value === 0) + throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error(`${name} must be a positive safe integer for bounded analysis`); + return value; +} + +export interface HostAnalysisResourceOptions { + totalMb?: number; + mainHeapMb?: number; + workers?: number; + includeParsing?: boolean; + /** Local model capacity is only reserved when this invocation may embed. */ + includeEmbeddings: boolean; + supervisorBytes?: number; + /** Share one host envelope among concurrent auto-sync analyses. */ + concurrency?: number; +} + +export function resolveHostAnalysisResources( + options: HostAnalysisResourceOptions, +): AnalysisResourceBudget { + const concurrency = options.concurrency ?? 1; + if (!Number.isSafeInteger(concurrency) || concurrency < 1) + throw new Error('Analysis concurrency must be a positive safe integer'); + const http = Boolean(process.env.GITNEXUS_EMBEDDING_URL && process.env.GITNEXUS_EMBEDDING_MODEL); + const effectiveMemoryBytes = Math.floor(effectiveRamBytes() / concurrency); + const request: AnalysisResourceRequest = { + effectiveMemoryBytes, + availableCpus: Math.max(1, Math.floor(os.availableParallelism() / concurrency)), + totalMb: options.totalMb, + mainHeapMb: options.mainHeapMb, + allowHeapOvercommit: options.mainHeapMb !== undefined && options.totalMb === undefined, + workers: options.workers ?? positiveEnvironment('GITNEXUS_WORKER_POOL_SIZE'), + workerHeapMb: positiveEnvironment('GITNEXUS_WORKER_HEAP_MB'), + embeddingThreads: options.includeEmbeddings + ? positiveEnvironment('GITNEXUS_EMBEDDING_THREADS') + : undefined, + preferredWorkers: 16, + includeParsing: options.includeParsing, + graphPoolBytes: positiveEnvironment('GITNEXUS_LBUG_BUFFER_POOL_SIZE'), + graphPoolGranuleRatio: granuleRatio(), + modelMemoryMb: options.includeEmbeddings && !http ? 2048 : 0, + supervisorBytes: options.supervisorBytes, + }; + return resolveAnalysisResourceBudget(request); +} + +/** Charge the retained frontend once, including a floor for its IPC/runtime. */ +export function analysisSupervisorReservation(): number { + return Math.max(192 * MIB, Math.ceil(process.memoryUsage().rss / MIB) * MIB); +} + +/** Cheap preflight only. Core still revalidates the exact slot under its lock. */ +export async function analysisMayEmbed( + repoPath: string, + options: { embeddings?: boolean; force?: boolean; dropEmbeddings?: boolean; branch?: string }, +): Promise { + if (options.embeddings) return true; + if (options.dropEmbeddings) return false; + const { getStoragePaths, loadMeta } = await import('../storage/repo-manager.js'); + const { resolveBranchPlacement } = await import('../storage/branch-index.js'); + const placement = await resolveBranchPlacement(repoPath, options.branch ?? null); + const directory = path.dirname(getStoragePaths(repoPath, placement.branch).metaPath); + const meta = await loadMeta(directory); + return Boolean( + meta?.embeddingJob || + meta?.embeddingCheckpoint || + (options.force && (meta?.stats?.embeddings ?? 0) > 0), + ); +} diff --git a/gitnexus/src/core/analysis-parse-cache.ts b/gitnexus/src/core/analysis-parse-cache.ts new file mode 100644 index 000000000..9f10c1c37 --- /dev/null +++ b/gitnexus/src/core/analysis-parse-cache.ts @@ -0,0 +1,258 @@ +/** Parser cache publication shared by an in-process writer and its exit supervisor. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import { + loadParseCache, + persistParseCacheChunk, + pruneCache, + saveParseCache, + type ParseCache, +} from '../storage/parse-cache.js'; +import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js'; +import { + embeddingJobDirectory, + openEmbeddingJobFile, + syncEmbeddingJobDirectory, +} from '../storage/embedding-job.js'; +import { withStoreLock } from '../storage/shared-store-lifecycle.js'; +import type { SharedStoreLayout } from '../storage/shared-store.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { listStoreMetaRoots } from './shared-store-analyze.js'; + +interface CachePublicationOptions { + storagePath: string; + metadataDirectory: string; + cacheRoot: string; + /** The exact published graph's handoff; never scan generation history. */ + publishedGeneration?: string; + sharedStore?: SharedStoreLayout; + collectBranchKeys: ( + root: string, + exclude: string, + ) => Promise<{ keys: Set; complete: boolean }>; + log: (message: string) => void; +} + +/** Called only after graph publication, under the ordinary index lock. */ +export async function publishAnalysisParseCache( + cache: ParseCache, + options: CachePublicationOptions, +): Promise { + const { storagePath, metadataDirectory, cacheRoot, sharedStore, collectBranchKeys, log } = + options; + const publish = async () => { + const listing = sharedStore + ? await listStoreMetaRoots(sharedStore) + : { roots: [storagePath], complete: true }; + const siblingKeys = new Set(); + let complete = listing.complete; + for (const root of listing.roots) { + const folded = await collectBranchKeys(root, metadataDirectory); + for (const key of folded.keys) siblingKeys.add(key); + if (!folded.complete) complete = false; + } + // Re-read inside the publication lock: another checkout can have saved + // since parsing began. Metadata alone never admits an incompatible shard. + const retained = await loadParseCache( + cacheRoot, + cache.identity, + cache.writeStoragePath ?? cacheRoot, + ); + cache.storagePath = cacheRoot; + cache.onDiskKeys = new Set(); + for (const key of retained.onDiskKeys ?? []) { + if ( + !cache.stagedKeys?.has(key) && + !cache.staleKeys?.has(key) && + (cache.usedKeys.has(key) || !complete || siblingKeys.has(key)) + ) + cache.onDiskKeys.add(key); + } + for (const [key, value] of retained.entries) { + if ( + !cache.usedKeys.has(key) && + !cache.stagedKeys?.has(key) && + !cache.staleKeys?.has(key) && + (!complete || siblingKeys.has(key)) + ) + cache.entries.set(key, value); + } + if (complete) { + for (const key of siblingKeys) cache.usedKeys.add(key); + } else { + log('Parse cache: a branch meta was unreadable — retaining all cached chunks (#2106).'); + for (const key of cache.entries.keys()) cache.usedKeys.add(key); + for (const key of cache.onDiskKeys) cache.usedKeys.add(key); + } + const pruned = pruneCache(cache, cache.usedKeys); + if (pruned > 0) log(`Parse cache: pruned ${pruned} stale chunk entries`); + const savedKeys = await saveParseCache(cacheRoot, cache); + await mergeStagedDurableParsedFileStore( + cacheRoot, + cache.writeStoragePath ?? cacheRoot, + cache.identity.base, + new Set(savedKeys), + ); + }; + await (sharedStore ? withStoreLock(sharedStore, 'cache', publish) : publish()); + if (options.publishedGeneration) { + const root = await embeddingJobDirectory(metadataDirectory); + await fs.rm(path.join(root, receiptName(options.publishedGeneration)), { force: true }); + await syncEmbeddingJobDirectory(root); + } +} + +// This contains only chunk keys, never ASTs, graph data or source content. +// The fixed ceiling fits the phase headroom; failure merely leaves this +// optional cache unpublished, and never changes graph/embedding correctness. +const MAX_HANDOFF_BYTES = 16 * 1024 * 1024; +const KEY = /^[a-f0-9]{64}$/; +interface CacheHandoff { + version: 1; + generation: string; + stage: string; + identity: ParseCache['identity']; + used: string[]; + staged: string[]; + stale: string[]; +} +const checksum = (payload: string) => createHash('sha256').update(payload).digest('hex'); +const receiptName = (generation: string) => { + if (!/^[a-f0-9]{32}$/.test(generation)) throw new Error('Invalid cache publication generation'); + return `analysis-cache-${generation}.json`; +}; + +async function ownedStage(root: string, relative: string): Promise { + if (!/^parse-rebuild\/run\.[A-Za-z0-9]{6}$/.test(relative)) + throw new Error('Invalid prepared parser cache stage'); + let current = await embeddingJobDirectory(root); + for (const part of relative.split('/')) + current = await embeddingJobDirectory(path.join(current, part)); + return current; +} + +/** Flush only memory fallback shards. Existing live shards stay in place; + * the supervisor will re-read their current identity under the cache lock. */ +export async function stageAnalysisParseCache( + root: string, + generation: string, + cache: ParseCache, +): Promise { + root = await embeddingJobDirectory(root); + const name = receiptName(generation); + if (!cache.writeStoragePath) + throw new Error('Prepared parser cache has no owned staging directory'); + const relative = path.relative(root, cache.writeStoragePath).split(path.sep).join('/'); + await ownedStage(root, relative); + // Bound each selection before copying or serializing it. Unknown/malformed + // keys cannot become filesystem paths. + for (const keys of [cache.usedKeys, cache.stagedKeys, cache.staleKeys]) { + if (!keys) continue; + if (keys.size > Math.floor(MAX_HANDOFF_BYTES / 3 / 70)) + throw new Error('Prepared parser cache key set exceeds its bound'); + for (const key of keys) + if (!KEY.test(key)) throw new Error('Prepared parser cache contains an invalid key'); + } + for (const [key, value] of cache.entries) await persistParseCacheChunk(cache, key, value); + if (cache.entries.size) + throw new Error('Could not persist prepared parser cache fallback shards'); + const payload: CacheHandoff = { + version: 1, + generation, + stage: relative, + identity: cache.identity, + used: [...cache.usedKeys], + staged: [...(cache.stagedKeys ?? [])], + stale: [...(cache.staleKeys ?? [])], + }; + const serialized = JSON.stringify(payload); + const bytes = Buffer.from(JSON.stringify({ payload, checksum: checksum(serialized) })); + if (bytes.length > MAX_HANDOFF_BYTES) + throw new Error('Prepared parser cache receipt exceeds its bound'); + const temporary = path.join(root, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(root, name)); + await syncEmbeddingJobDirectory(root); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +/** The published graph binds the generation, parser identity and used keys. + * Missing cache state is optional; malformed state is never used as a cache. */ +export async function loadAnalysisParseCache( + root: string, + metadata: RepoMeta, +): Promise { + const generation = metadata.graphGeneration?.id; + if (!generation) throw new Error('Cache publication requires a published graph generation'); + root = await embeddingJobDirectory(root); + const file = path.join(root, receiptName(generation)); + let opened; + try { + opened = await openEmbeddingJobFile(file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + const { handle, stat } = opened; + let bytes: Buffer; + try { + if (stat.size > MAX_HANDOFF_BYTES) throw new Error('Unverified prepared parser cache receipt'); + bytes = Buffer.alloc(stat.size + 1); + let length = 0; + while (length < bytes.length) { + const result = await handle.read(bytes, length, bytes.length - length, length); + if (!result.bytesRead) break; + length += result.bytesRead; + } + if (length !== stat.size) throw new Error('Prepared parser cache receipt changed'); + bytes = bytes.subarray(0, length); + } finally { + await handle.close(); + } + const envelope = JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(bytes)); + const payload = envelope.payload as CacheHandoff; + const keys = (value: unknown): value is string[] => + Array.isArray(value) && value.every((key) => typeof key === 'string' && KEY.test(key)); + if ( + !payload || + envelope.checksum !== checksum(JSON.stringify(payload)) || + payload.version !== 1 || + payload.generation !== generation || + typeof payload.stage !== 'string' || + !payload.identity || + typeof payload.identity.base !== 'string' || + typeof payload.identity.pdg !== 'string' || + payload.identity.base !== metadata.parseCacheIdentity?.base || + (metadata.parseCacheIdentity.pdg && + payload.identity.pdg !== metadata.parseCacheIdentity.pdg.identity) || + !keys(payload.used) || + !keys(payload.staged) || + !keys(payload.stale) + ) + throw new Error('Prepared parser cache does not match the published graph'); + const usedKeys = new Set(metadata.cacheKeys ?? []); + if ( + payload.used.length !== usedKeys.size || + new Set(payload.used).size !== usedKeys.size || + payload.used.some((key) => !usedKeys.has(key)) + ) + throw new Error('Prepared parser cache selection differs from the published graph'); + return { + identity: payload.identity, + entries: new Map(), + usedKeys, + writeStoragePath: await ownedStage(root, payload.stage), + stagedKeys: new Set(payload.staged), + staleKeys: new Set(payload.stale), + }; +} diff --git a/gitnexus/src/core/analysis-phase-supervisor.ts b/gitnexus/src/core/analysis-phase-supervisor.ts new file mode 100644 index 000000000..c88f1886e --- /dev/null +++ b/gitnexus/src/core/analysis-phase-supervisor.ts @@ -0,0 +1,156 @@ +/** Filesystem-only handoff from an exit-only graph writer. No model/native imports. */ +import path from 'node:path'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; +import { embeddingJobDirectory } from '../storage/embedding-job.js'; +import { loadMeta } from '../storage/repo-meta.js'; +import { + publishPreparedAnalysis, + readPreparedAnalysis, + type AnalysisPublicationActions, +} from './analysis-publication.js'; + +export interface AnalysisPhaseTarget { + repoPath: string; + storagePath: string; + metadataDirectory: string; + repoName: string; + commit: string; +} + +/** Only this bounded receipt crosses IPC; the live graph stays in its child. */ +export interface PreparedAnalysisHandoff extends AnalysisPhaseTarget { + version: 1; + publicationId: string; +} + +export function projectPreparedAnalysisHandoff(value: unknown): PreparedAnalysisHandoff { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid prepared analysis handoff'); + const input = value as Record; + const text = (name: string, maximum: number, allowEmpty = false): string => { + const value = input[name]; + if (typeof value !== 'string' || (!allowEmpty && !value.length) || value.length > maximum) + throw new Error(`Invalid prepared analysis ${name}`); + return value; + }; + const projected: PreparedAnalysisHandoff = { + version: 1, + publicationId: text('publicationId', 32), + repoPath: text('repoPath', 32768), + storagePath: text('storagePath', 32768), + metadataDirectory: text('metadataDirectory', 32768), + repoName: text('repoName', 1024), + commit: text('commit', 64, true), + }; + if ( + input.version !== 1 || + !/^[a-f0-9]{32}$/.test(projected.publicationId) || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})?$/.test(projected.commit) || + ![projected.repoPath, projected.storagePath, projected.metadataDirectory].every(path.isAbsolute) + ) + throw new Error('Invalid prepared analysis identity or location'); + return projected; +} + +export interface AnalysisPhasePublicationActions extends AnalysisPublicationActions { + /** Re-resolve checkout placement, current HEAD and the registered name. */ + resolveTarget: () => Promise; + /** Finish registry/shared-store/cache bookkeeping and reader refresh. Called + * under the ordinary slot lock after native verification and publication. + * A cancellation after publication must still leave this graph available. */ + finalizeGraph: (metadata: RepoMeta) => Promise; + signal?: AbortSignal; + log?: (message: string) => void; + onWaitStart?: () => void; +} + +function requireSameTarget(expected: AnalysisPhaseTarget, actual: AnalysisPhaseTarget): void { + if ( + expected.repoName !== actual.repoName || + expected.commit !== actual.commit || + (['repoPath', 'storagePath', 'metadataDirectory'] as const).some( + (key) => path.resolve(expected[key]) !== path.resolve(actual[key]), + ) + ) + throw new Error( + 'Analysis slot, checkout or registered name changed during the graph handoff; retry before embedding', + ); +} + +/** The launcher calls this only after its actual-exit promise resolves. The + * persisted writer-PID guard independently rejects an early or stale handoff. + * No borrowed lock: acquire normally, re-resolve under ownership, then finish + * the exact prepared attempt. An embedding continuation must acquire its own + * lock and recheck this returned generation before making paid requests. */ +export async function publishAnalysisAfterProcessExit( + handoff: PreparedAnalysisHandoff, + writerPid: number, + actions: AnalysisPhasePublicationActions, +): Promise { + const expected = projectPreparedAnalysisHandoff(handoff); + if (!Number.isSafeInteger(writerPid) || writerPid < 1) + throw new Error('Missing graph writer PID'); + actions.signal?.throwIfAborted(); + requireSameTarget(expected, await actions.resolveTarget()); + const root = await embeddingJobDirectory(expected.metadataDirectory); + const lock = await acquireAnalysisSlotLock(root, { + sweep: false, + log: actions.log, + onWaitStart: actions.onWaitStart, + }); + try { + requireExclusiveIndexLock( + lock, + 'Graph publication requires exclusive ownership of its index slot', + ); + actions.signal?.throwIfAborted(); + requireSameTarget(expected, await actions.resolveTarget()); + const prepared = await readPreparedAnalysis(root); + if (prepared && (prepared.id !== expected.publicationId || prepared.writerPid !== writerPid)) + throw new Error( + 'Prepared graph attempt changed during handoff; artifacts retained; retry analysis', + ); + const metadata = prepared + ? await publishPreparedAnalysis(root, { + saveMetadata: actions.saveMetadata, + verify: async (file, metadata) => { + if (metadata.repoPath !== expected.repoPath || metadata.lastCommit !== expected.commit) + throw new Error('Prepared graph belongs to a different checkout'); + actions.signal?.throwIfAborted(); + await actions.verify(file, metadata); + actions.signal?.throwIfAborted(); + }, + }) + : await loadMeta(root); + // Another waiting writer may recover this exact receipt after the graph + // child exits. Adopt only the clean, already-published generation. + if ( + !metadata || + metadata.graphGeneration?.id !== expected.publicationId || + metadata.repoPath !== expected.repoPath || + metadata.lastCommit !== expected.commit || + metadata.incrementalInProgress || + metadata.graphWriteCollapsed + ) + throw new Error( + 'Prepared graph attempt changed during handoff; artifacts retained; retry analysis', + ); + // Once publication starts, complete discoverability even if cancellation + // arrived during its filesystem operations. Paid work remains inadmissible. + const finalized = await actions.finalizeGraph(metadata); + requireSameTarget(expected, await actions.resolveTarget()); + if ( + finalized.graphGeneration?.id !== expected.publicationId || + finalized.repoPath !== expected.repoPath || + finalized.incrementalInProgress || + finalized.graphWriteCollapsed + ) + throw new Error('Graph finalization did not retain the published generation'); + actions.signal?.throwIfAborted(); + return finalized; + } finally { + lock.release(); + } +} diff --git a/gitnexus/src/core/analysis-progress.ts b/gitnexus/src/core/analysis-progress.ts new file mode 100644 index 000000000..f56f5021a --- /dev/null +++ b/gitnexus/src/core/analysis-progress.ts @@ -0,0 +1,156 @@ +import type { EmbeddingJobState } from '../storage/embedding-job-journal.js'; + +export type GraphAnalysisPhase = 'pending' | 'building' | 'ready' | 'failed'; + +/** Counts describe durable acknowledgements. Active work is part of pending, + * never an additional category. Totals remain absent before manifest preflight. */ +export interface EmbeddingJobProgress { + phase: 'preparing' | 'embedding' | 'indexing' | 'ready' | 'preserved' | 'failed' | 'unverified'; + totalUnits?: number; + completedUnits?: number; + reusedUnits?: number; + pendingUnits?: number; + failedUnits?: number; + activeUnit?: number | null; + totalNodes?: number; + completedNodes?: number; + reusedNodes?: number; + pendingNodes?: number; + failedNodes?: number; + excludedNodes?: number; + chunks?: number; +} + +/** Additive fields shared by CLI callbacks, worker IPC, and HTTP progress. */ +export interface AnalysisPhaseProgress { + graphPhase?: GraphAnalysisPhase; + embeddingProgress?: EmbeddingJobProgress; + modelDownloadPercent?: number; +} + +const publicEmbeddingProgress = ( + progress: EmbeddingJobProgress | undefined, +): EmbeddingJobProgress | undefined => { + if ( + !progress || + !['preparing', 'embedding', 'indexing', 'ready', 'preserved', 'failed', 'unverified'].includes( + progress.phase, + ) + ) + return undefined; + const safe: EmbeddingJobProgress = { phase: progress.phase }; + // Explicit numeric allowlist: the source may also contain internal paths, + // identities, credentials, or values that cannot cross JSON IPC. + for (const key of [ + 'totalUnits', + 'completedUnits', + 'reusedUnits', + 'pendingUnits', + 'failedUnits', + 'totalNodes', + 'completedNodes', + 'reusedNodes', + 'pendingNodes', + 'failedNodes', + 'excludedNodes', + 'chunks', + ] as const) { + const value = progress[key]; + if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) safe[key] = value; + } + if ( + progress.activeUnit === null || + (typeof progress.activeUnit === 'number' && + Number.isSafeInteger(progress.activeUnit) && + progress.activeUnit >= 0) + ) + safe.activeUnit = progress.activeUnit; + return safe; +}; + +/** The same allowlist protects worker IPC and public progress feeds. */ +export function projectAnalysisPhaseProgress( + progress: AnalysisPhaseProgress | undefined, +): AnalysisPhaseProgress { + const result: AnalysisPhaseProgress = {}; + if (!progress) return result; + if ( + progress.graphPhase && + ['pending', 'building', 'ready', 'failed'].includes(progress.graphPhase) + ) + result.graphPhase = progress.graphPhase; + const embeddingProgress = publicEmbeddingProgress(progress.embeddingProgress); + if (embeddingProgress) result.embeddingProgress = embeddingProgress; + if ( + typeof progress.modelDownloadPercent === 'number' && + Number.isFinite(progress.modelDownloadPercent) && + progress.modelDownloadPercent >= 0 && + progress.modelDownloadPercent <= 100 + ) + result.modelDownloadPercent = progress.modelDownloadPercent; + return result; +} + +export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJobProgress { + const counts = [ + state.totalUnits, + state.completedUnits, + state.reusedUnits, + state.failedUnits, + state.totalNodes, + state.completedNodes, + state.reusedNodes, + state.failedNodes, + state.excludedNodes ?? 0, + state.chunks, + ]; + const pendingUnits = + state.totalUnits - state.completedUnits - state.reusedUnits - state.failedUnits; + const pendingNodes = + state.totalNodes - + state.completedNodes - + state.reusedNodes - + state.failedNodes - + (state.excludedNodes ?? 0); + const activeUnit = state.active?.start?.unit ?? null; + if ( + counts.some((n) => !Number.isSafeInteger(n) || n < 0) || + pendingUnits < 0 || + pendingNodes < 0 || + state.active === undefined || + !['embedding', 'indexing', 'ready'].includes(state.phase) || + (state.active !== null && + (!Number.isSafeInteger(activeUnit) || + activeUnit === null || + activeUnit < 0 || + activeUnit >= state.totalUnits || + pendingUnits < 1 || + !Number.isSafeInteger(state.active.nodes) || + state.active.nodes < 1 || + state.active.nodes > pendingNodes)) || + (state.phase !== 'embedding' && + (pendingUnits || + pendingNodes || + state.failedUnits || + state.failedNodes || + state.active !== null)) + ) { + throw new Error('Embedding progress accounting is unverified'); + } + return { + phase: state.phase, + totalUnits: state.totalUnits, + completedUnits: state.completedUnits, + reusedUnits: state.reusedUnits, + pendingUnits, + failedUnits: state.failedUnits, + activeUnit, + totalNodes: state.totalNodes, + completedNodes: state.completedNodes, + reusedNodes: state.reusedNodes, + pendingNodes, + failedNodes: state.failedNodes, + ...(state.excludedNodes ? { excludedNodes: state.excludedNodes } : {}), + chunks: state.chunks, + }; +} diff --git a/gitnexus/src/core/analysis-publication.ts b/gitnexus/src/core/analysis-publication.ts new file mode 100644 index 000000000..e2bf88146 --- /dev/null +++ b/gitnexus/src/core/analysis-publication.ts @@ -0,0 +1,487 @@ +/** Durable graph publication. Native verification/close and metadata ownership + * are supplied by the coordinator, which holds the index-directory write lock. */ +import fs from 'node:fs/promises'; +import { constants } from 'node:fs'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from '../storage/storage-constants.js'; +import { preserveVectorCoverage } from '../storage/vector-generation.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js'; +import { readProcStartTime } from '../storage/index-lock.js'; +import { reclaimAnalysisArtifacts } from '../storage/analysis-artifact-cleanup.js'; + +const RECEIPT = 'analysis-publication.json'; +const GENERATION = /^[a-f0-9]{32}$/; +const PRIVATE_GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/; +const DIGEST = /^[a-f0-9]{64}$/; +const STAGING = /^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; +const MAX_RECEIPT_BYTES = 4096; +const MAX_METADATA_BYTES = 256 * 1024 * 1024; +const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); +// Share the incarnation across loaders of this module within one process. +const INSTANCE_KEY = Symbol.for('gitnexus.analysis-publication.writer-instance'); +const processIdentity = process as unknown as Record; +const WRITER_INSTANCE = (processIdentity[INSTANCE_KEY] ??= randomUUID().replaceAll('-', '')); + +export interface PreparedAnalysisPublication { + version: 2; + mode: 'staged' | 'in-place'; + id: string; + /** Source basename; also names the existing file for in-place writes. */ + stagingFile: string; + graphFile: string; + metadataFile: string; + metadataBytes: number; + metadataDigest: string; + /** Exact prior metadata, or absence, observed under the slot's write lock. */ + priorMetadataDigest: string | null; + /** Permitted intermediate state while retiring a canonical legacy graph. */ + redirectedMetadataDigest?: string; + /** Exit-only callers may not publish while this process is still alive. */ + writerPid?: number; + writerStartTime?: string; + writerInstance?: string; + /** A hard-link destination for the previous legacy graph; never copied. */ + legacyGraphFile?: string; +} + +export type AnalysisPublicationOptions = ( + | { mode?: 'staged' } + | { mode: 'in-place'; attemptId: string } +) & { waitForProcessExit?: boolean }; + +export interface AnalysisPublicationActions { + /** Open read-only and verify graph/schema/FTS against the prepared metadata. + * Must close the verification handle before returning (including on Windows). + * Unknown counts/catalog state must reject, never certify an empty graph. + */ + verify: (graphPath: string, metadata: RepoMeta) => Promise; + /** The normal metadata writer, including its compatibility mirror. */ + saveMetadata: (metadata: RepoMeta) => Promise; +} + +function validateReceipt(value: unknown): asserts value is PreparedAnalysisPublication { + const r = value as PreparedAnalysisPublication; + if ( + !r || + r.version !== 2 || + !GENERATION.test(r.id) || + !PRIVATE_GRAPH.test(r.graphFile) || + (r.mode === 'staged' + ? !STAGING.test(r.stagingFile) || r.graphFile !== `lbug.generation.${r.id}` + : r.mode !== 'in-place' || + (r.stagingFile === 'lbug' + ? r.graphFile !== `lbug.generation.${r.id}` + : !PRIVATE_GRAPH.test(r.stagingFile) || r.graphFile !== r.stagingFile)) || + r.metadataFile !== `analysis-meta-${r.id}.json` || + (r.priorMetadataDigest !== null && !DIGEST.test(r.priorMetadataDigest)) || + (r.redirectedMetadataDigest !== undefined && + (!r.legacyGraphFile || !DIGEST.test(r.redirectedMetadataDigest))) || + (r.writerPid !== undefined && (!Number.isSafeInteger(r.writerPid) || r.writerPid < 1)) || + (r.writerStartTime !== undefined && (!r.writerPid || !/^\d{1,32}$/.test(r.writerStartTime))) || + (r.writerInstance !== undefined && (!r.writerPid || !GENERATION.test(r.writerInstance))) || + (r.legacyGraphFile !== undefined && + (r.mode !== 'staged' || + !PRIVATE_GRAPH.test(r.legacyGraphFile) || + r.legacyGraphFile === r.graphFile)) || + !Number.isSafeInteger(r.metadataBytes) || + r.metadataBytes < 1 || + r.metadataBytes > MAX_METADATA_BYTES || + !DIGEST.test(r.metadataDigest) + ) + throw new Error('Invalid prepared graph publication; artifacts retained'); +} + +async function regularFile(file: string): Promise { + try { + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Publication artifact must be an owned regular file'); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } +} + +async function boundedRead(file: string, maximum: number): Promise { + const handle = await fs.open( + file, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); + try { + const stat = await handle.stat(); + const entry = await fs.lstat(file); + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Publication artifact must be an owned regular file'); + if (stat.dev !== entry.dev || stat.ino !== entry.ino) + throw new Error('Publication artifact changed while opening'); + if (!stat.isFile() || stat.size > maximum) + throw new Error('Publication artifact exceeds its receipt bound'); + const buffer = Buffer.alloc(stat.size + 1); + let size = 0; + while (size < buffer.length) { + const result = await handle.read(buffer, size, buffer.length - size, size); + if (!result.bytesRead) break; + size += result.bytesRead; + } + if (size !== stat.size) throw new Error('Publication artifact changed while reading'); + return buffer.subarray(0, size); + } finally { + await handle.close(); + } +} + +async function writeDurable(directory: string, name: string, bytes: Buffer): Promise { + const temporary = path.join(directory, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(directory, name)); + await syncEmbeddingJobDirectory(directory); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +async function readSlotMetadata(root: string): Promise { + try { + let bytes: Buffer; + try { + bytes = await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + bytes = await boundedRead(path.join(root, LEGACY_METADATA_FILE), MAX_METADATA_BYTES); + } + const metadata: RepoMeta = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + if (!metadata || typeof metadata.repoPath !== 'string') + throw new Error('Invalid graph slot metadata; artifacts retained'); + return metadata; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +const metadataDigest = (metadata: RepoMeta | null): string | null => + metadata === null ? null : hash(JSON.stringify(metadata)); + +/** Replaces the existing pre-write dirty stamp, never an extra post-write stamp. + * The coordinator holds the slot lock and must await this before mutating native + * data. An interrupted in-place attempt requires the usual dirty-graph rebuild. + */ +export async function beginInPlaceAnalysis( + directory: string, + metadata: RepoMeta, + saveMetadata: AnalysisPublicationActions['saveMetadata'], +): Promise { + const root = await embeddingJobDirectory(directory); + if ( + !metadata.incrementalInProgress || + (metadata.graphPath !== undefined && !PRIVATE_GRAPH.test(metadata.graphPath)) + ) + throw new Error('In-place publication requires a private graph and a dirty write plan'); + if (await readPreparedAnalysis(root)) + throw new Error('Recover the pending graph publication before starting another'); + // Refuse an aliased native target before the caller can perform its first write. + await regularFile(path.join(root, metadata.graphPath ?? 'lbug')); + const prior = await readSlotMetadata(root); + if (prior && prior.repoPath !== metadata.repoPath) + throw new Error('Graph slot belongs to another repository'); + const attemptId = randomUUID().replaceAll('-', ''); + const dirty: RepoMeta = { + ...metadata, + vectorGeneration: preserveVectorCoverage(metadata, attemptId), + incrementalInProgress: { + ...metadata.incrementalInProgress, + writePlan: 'in-place', + attemptId, + }, + }; + delete dirty.graphGeneration; + await saveMetadata(dirty); + await syncPublishedMetadata(root, dirty); + return dirty; +} + +/** Call after graph/FTS verification and a successful native checkpoint, before + * native close. A close crash leaves the owned build and its publication intent. + * No model or vector work may start until publishPreparedAnalysis completes. + */ +export async function prepareAnalysisPublication( + directory: string, + buildPath: string, + metadata: RepoMeta, + coveredContent: string, + options: AnalysisPublicationOptions = {}, +): Promise { + const root = await embeddingJobDirectory(directory); + if ( + !DIGEST.test(coveredContent) || + !metadata.repoPath || + !metadata.runnerIdentity || + metadata.incrementalInProgress || + metadata.graphWriteCollapsed + ) + throw new Error('Graph publication requires verified source and analyzer metadata'); + const mode = options.mode ?? 'staged'; + const sourceFile = path.basename(buildPath); + if ( + (await fs.realpath(path.dirname(buildPath))) !== root || + (mode === 'staged' + ? !STAGING.test(sourceFile) + : sourceFile !== 'lbug' && !PRIVATE_GRAPH.test(sourceFile)) + ) + throw new Error('Graph build is outside its owned staging directory'); + if (!(await regularFile(buildPath))) throw new Error('Graph build is missing'); + if (await regularFile(path.join(root, RECEIPT))) + throw new Error('Recover the pending graph publication before preparing another'); + const prior = await readSlotMetadata(root); + if (prior && prior.repoPath !== metadata.repoPath) + throw new Error('Graph slot belongs to another repository'); + if ( + options.mode === 'in-place' && + (!GENERATION.test(options.attemptId) || + prior?.incrementalInProgress?.attemptId !== options.attemptId || + (prior.graphPath ?? 'lbug') !== sourceFile || + prior.graphGeneration) + ) + throw new Error('In-place graph attempt is missing or superseded; artifacts retained'); + const id = options.mode === 'in-place' ? options.attemptId : randomUUID().replaceAll('-', ''); + const graphFile = + mode === 'in-place' && sourceFile !== 'lbug' ? sourceFile : `lbug.generation.${id}`; + const metadataFile = `analysis-meta-${id}.json`; + const preparedMeta: RepoMeta = { + ...metadata, + vectorGeneration: preserveVectorCoverage(metadata, id), + graphPath: graphFile, + graphGeneration: { version: 1, id, coveredContent }, + }; + const bytes = Buffer.from(JSON.stringify(preparedMeta)); + if (bytes.length > MAX_METADATA_BYTES) + throw new Error('Graph metadata exceeds the supported publication size'); + const receipt: PreparedAnalysisPublication = { + version: 2, + mode, + id, + stagingFile: path.basename(buildPath), + graphFile, + metadataFile, + metadataBytes: bytes.length, + metadataDigest: hash(bytes), + priorMetadataDigest: metadataDigest(prior), + ...(options.waitForProcessExit + ? { + writerPid: process.pid, + writerInstance: WRITER_INSTANCE, + writerStartTime: readProcStartTime(process.pid) ?? undefined, + } + : {}), + ...(mode === 'staged' && (await regularFile(path.join(root, 'lbug'))) + ? { legacyGraphFile: `lbug.generation.${randomUUID().replaceAll('-', '')}` } + : {}), + }; + if (receipt.legacyGraphFile && prior && !prior.graphPath) + receipt.redirectedMetadataDigest = hash( + JSON.stringify({ ...prior, graphPath: receipt.legacyGraphFile }), + ); + validateReceipt(receipt); + await writeDurable(root, metadataFile, bytes); + const payload = JSON.stringify(receipt); + await writeDurable( + root, + RECEIPT, + Buffer.from(JSON.stringify({ checksum: hash(payload), payload: receipt })), + ); + return receipt; +} + +export async function readPreparedAnalysis( + directory: string, +): Promise { + const root = await embeddingJobDirectory(directory); + let bytes: Buffer; + try { + bytes = await boundedRead(path.join(root, RECEIPT), MAX_RECEIPT_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + const envelope: unknown = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + if ( + !envelope || + typeof envelope !== 'object' || + !('payload' in envelope) || + !('checksum' in envelope) || + hash(JSON.stringify(envelope.payload)) !== envelope.checksum + ) + throw new Error('Prepared publication checksum mismatch; artifacts retained'); + validateReceipt(envelope.payload); + return envelope.payload; +} + +export async function syncPublishedMetadata(root: string, metadata: RepoMeta): Promise { + const metadataPath = path.join(root, INDEX_METADATA_FILE); + const saved = await boundedRead(metadataPath, MAX_METADATA_BYTES); + if ( + hash(JSON.stringify(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(saved)))) !== + hash(JSON.stringify(metadata)) + ) + throw new Error( + 'Published graph metadata does not match the prepared generation; artifacts retained', + ); + // Windows FlushFileBuffers requires a writable descriptor. The coordinator + // owns the write lock; this opens the file without changing its contents. + const handle = await fs.open(metadataPath, constants.O_RDWR | (constants.O_NOFOLLOW ?? 0)); + try { + await handle.sync(); + } finally { + await handle.close(); + } + await syncEmbeddingJobDirectory(root); +} + +/** Older binaries ignore private generation pointers. Remove the obsolete + * canonical name before advancing metadata, retaining its inode for readers + * and rollback. A hard-link failure leaves the old index readable and aborts; + * relocating vectors must never trigger a full graph copy. + */ +async function retireLegacyGraph( + root: string, + receipt: PreparedAnalysisPublication, + metadata: RepoMeta, + actions: AnalysisPublicationActions, + previous: RepoMeta | null, +): Promise { + const legacy = path.join(root, 'lbug'); + if (!receipt.legacyGraphFile) { + if (await regularFile(legacy)) + throw new Error('Unexpected legacy graph appeared during publication'); + return; + } + const retired = path.join(root, receipt.legacyGraphFile); + if (!(await regularFile(legacy))) { + if (!(await regularFile(retired))) throw new Error('Legacy graph retirement artifact missing'); + return; + } + if (await regularFile(retired)) { + const [before, after] = await Promise.all([fs.stat(legacy), fs.stat(retired)]); + if (!before.ino || before.ino !== after.ino || before.dev !== after.dev) + throw new Error('Conflicting legacy graph retirement artifact'); + } else { + await fs.link(legacy, retired); + await syncEmbeddingJobDirectory(root); + } + if (previous && previous.repoPath !== metadata.repoPath) + throw new Error('Legacy graph belongs to another repository'); + if (previous && !previous.graphPath) { + const redirected = { ...previous, graphPath: receipt.legacyGraphFile }; + await actions.saveMetadata(redirected); + await syncPublishedMetadata(root, redirected); + } + try { + await fs.unlink(legacy); + } catch (error) { + if (['EBUSY', 'EPERM', 'EACCES'].includes((error as NodeJS.ErrnoException).code ?? '')) + throw new Error( + 'Cannot retire the legacy graph while a reader holds it. Stop older MCP/serve processes and rerun analyze; the prepared graph and embeddings are retained.', + { cause: error }, + ); + throw error; + } + await syncEmbeddingJobDirectory(root); +} + +/** Also recovers crashes after rename or metadata publication. Every attempt + * re-verifies the owned native artifact; a generation identifier is not proof. + * Staged writes retain the prior reader artifact. In-place writes have already + * invalidated it and publish a new logical generation on the same private file. + */ +export async function publishPreparedAnalysis( + directory: string, + actions: AnalysisPublicationActions, +): Promise { + const root = await embeddingJobDirectory(directory); + const receipt = await readPreparedAnalysis(root); + if (!receipt) return undefined; + if (receipt.writerPid !== undefined) { + const deadline = performance.now() + 2000; + for (;;) { + // A live, recycled PID is not the process that prepared this receipt. + if ( + receipt.writerPid === process.pid && + receipt.writerInstance && + receipt.writerInstance !== WRITER_INSTANCE + ) + break; + const started = readProcStartTime(receipt.writerPid); + if (receipt.writerStartTime && started && receipt.writerStartTime !== started) break; + try { + process.kill(receipt.writerPid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') break; + throw new Error( + `Cannot verify graph writer PID ${receipt.writerPid}: ${error instanceof Error ? error.message : String(error)}. Run analyze as the original index owner after that process exits; artifacts retained.`, + { cause: error }, + ); + } + if (performance.now() >= deadline) + throw new Error( + `Graph writer PID ${receipt.writerPid} has not exited; publication must wait for physical release. Stop that analysis process and retry; artifacts retained.`, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + const bytes = await boundedRead(path.join(root, receipt.metadataFile), receipt.metadataBytes); + if (bytes.length !== receipt.metadataBytes || hash(bytes) !== receipt.metadataDigest) + throw new Error('Prepared graph metadata checksum mismatch; artifacts retained'); + const metadata = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as RepoMeta; + if ( + metadata.graphPath !== receipt.graphFile || + metadata.graphGeneration?.version !== 1 || + metadata.graphGeneration.id !== receipt.id || + !DIGEST.test(metadata.graphGeneration.coveredContent) + ) + throw new Error('Prepared graph generation mismatch'); + const previous = await readSlotMetadata(root); + const currentDigest = metadataDigest(previous); + if ( + currentDigest !== receipt.priorMetadataDigest && + currentDigest !== receipt.redirectedMetadataDigest && + currentDigest !== receipt.metadataDigest + ) + throw new Error( + 'Prepared graph publication was superseded; artifacts retained; retry analysis', + ); + const staged = path.join(root, receipt.stagingFile); + const published = path.join(root, receipt.graphFile); + const stagedExists = await regularFile(staged); + const publishedExists = await regularFile(published); + const retainedInPlace = staged === published; + if (retainedInPlace ? !publishedExists : stagedExists === publishedExists) + throw new Error('Prepared publication has missing or conflicting graph artifacts'); + const source = stagedExists ? staged : published; + await actions.verify(source, metadata); + if (stagedExists && !retainedInPlace) { + await fs.rename(staged, published); + await syncEmbeddingJobDirectory(root); + } + await retireLegacyGraph(root, receipt, metadata, actions, previous); + await actions.saveMetadata(metadata); + // The canonical pointer must be durable before callers start paid work. + await syncPublishedMetadata(root, metadata); + await fs.unlink(path.join(root, RECEIPT)); + // Persist removal of the recovery pointer before removing what it names. + // A power loss must not resurrect a receipt whose metadata is already gone. + await syncEmbeddingJobDirectory(root); + await fs.unlink(path.join(root, receipt.metadataFile)); + await syncEmbeddingJobDirectory(root); + await reclaimAnalysisArtifacts(root, metadata, previous?.graphPath ?? receipt.legacyGraphFile); + return metadata; +} diff --git a/gitnexus/src/core/analysis-resource-budget.ts b/gitnexus/src/core/analysis-resource-budget.ts new file mode 100644 index 000000000..657c6ed46 --- /dev/null +++ b/gitnexus/src/core/analysis-resource-budget.ts @@ -0,0 +1,293 @@ +import { heapCapMbFor } from './ingestion/utils/effective-ram.js'; +/** Pure admission policy. Reservations are not an OS-enforced RSS limit. */ +const MIB = 1024 * 1024; +const GRAPH_POOL_MB = 256; +const VECTOR_POOL_MB = 64; +const MIN_MAIN_HEAP_MB = 384; +const MIN_WORKER_HEAP_MB = 512; +const WORKER_NATIVE_MB = 128; + +export interface AnalysisResourceRequest { + /** Effective process/container RAM, before reserving host headroom. */ + effectiveMemoryBytes: number; + availableCpus: number; + /** Explicit total reservation ceiling; independent of the heap-only flag. */ + totalMb?: number; + mainHeapMb?: number; + /** Honor an existing heap override when no explicit family ceiling was set. + * This may reserve virtual memory above RAM; native defaults still use RAM. */ + allowHeapOvercommit?: boolean; + workers?: number; + workerHeapMb?: number; + embeddingThreads?: number; + /** Existing automatic worker policy; explicit workers still wins. */ + preferredWorkers?: number; + /** Resume/sync can admit embeddings without reserving parser workers. */ + includeParsing?: boolean; + graphPoolBytes?: number; + /** LadybugDB COPY accounting uses OS-page-sized granules. */ + graphPoolGranuleRatio?: number; + /** Resident parent process, charged once per child phase. */ + supervisorBytes?: number; + /** Estimated peak model/sidecar allocation; zero for an HTTP provider. */ + modelMemoryMb: number; +} + +export interface AnalysisResourceBudget { + readonly totalBytes: number; + readonly automatic: boolean; + readonly headroomBytes: number; + readonly supervisorBytes: number; + readonly mainHeapMb: number; + readonly parse: { + readonly workers: number; + readonly workerHeapMb: number; + readonly workerNativeBytes: number; + readonly graphPoolBytes: number; + /** Graph writes run after parser workers drain and may use the full CPU allowance. */ + readonly nativeThreads: number; + readonly reservedBytes: number; + }; + readonly embedding: { + readonly graphPoolBytes: number; + readonly vectorPoolBytes: number; + readonly modelBytes: number; + /** Sidecar old-generation allocation, included in modelBytes with native/young space. */ + readonly modelHeapMb: number; + /** Combined source, expanded text, encoded request, vectors and prefetch. */ + readonly queueBytes: number; + readonly nativeThreads: number; + readonly reservedBytes: number; + }; +} + +function integer(value: number, label: string, minimum = 1): number { + if (!Number.isSafeInteger(value) || value < minimum) { + throw new Error(`${label} must be a safe integer of at least ${minimum}`); + } + return value; +} + +function bytes(mb: number, label: string): number { + return integer(integer(mb, label, 0) * MIB, `${label} in bytes`, 0); +} + +export const UNBOUNDED_GRAPH_POOL_MESSAGE = + "Zero graph pool bytes selects LadybugDB's native 80%-of-RAM default, which bounded analysis cannot reserve. " + + 'Unset GITNEXUS_LBUG_BUFFER_POOL_SIZE for automatic sizing or set a positive byte limit of at least 8388608.'; + +/** Resolve before opening a writable database or starting parser/model work. */ +export function resolveAnalysisResourceBudget( + request: AnalysisResourceRequest, +): AnalysisResourceBudget { + if (request.graphPoolBytes === 0) throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE); + const memory = integer(request.effectiveMemoryBytes, 'effectiveMemoryBytes'); + const cpus = integer(request.availableCpus, 'availableCpus'); + const embeddingThreads = + request.embeddingThreads === undefined + ? Math.min(4, cpus) + : integer(request.embeddingThreads, 'embedding threads'); + if (embeddingThreads > cpus) throw new Error('Embedding threads exceed the CPU allowance'); + const availableMb = Math.floor(memory / MIB); + const automatic = request.totalMb === undefined; + let totalMb = + request.totalMb === undefined + ? Math.floor(availableMb * 0.75) + : integer(request.totalMb, 'resource budget'); + if (totalMb > availableMb) { + throw new Error( + `Resource budget ${totalMb} MiB exceeds effective process memory ${availableMb} MiB`, + ); + } + const headroomMb = Math.max(128, Math.ceil(totalMb * 0.12)); + const supervisorMb = Math.ceil( + integer(request.supervisorBytes ?? 0, 'supervisor reservation', 0) / MIB, + ); + const poolScale = integer(request.graphPoolGranuleRatio ?? 1, 'graph pool granule ratio'); + const graphPoolBytes = + request.graphPoolBytes === undefined + ? bytes( + automatic + ? Math.min(2048 * poolScale, Math.floor(availableMb * 0.8)) + : Math.max(GRAPH_POOL_MB * poolScale, Math.min(4096, Math.floor(totalMb * 0.15))), + 'graph pool', + ) + : integer(request.graphPoolBytes, 'graph pool bytes', 8 * MIB); + // Zero is the native engine's unbounded override, never a bounded allocation. + const graphPoolMb = Math.ceil(graphPoolBytes / MIB); + const vectorPoolMb = Math.max(VECTOR_POOL_MB, Math.min(4096, Math.floor(totalMb / 8))); + const modelMb = integer(request.modelMemoryMb, 'model reservation', 0); + if (modelMb > 0 && modelMb < 512) + throw new Error( + 'Local model reservation must be at least 512 MiB for heap and native allocations', + ); + // A supported 10 MiB source node needs its full text/AST working set. Larger + // explicit budgets must be able to admit it without truncating its content. + const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32))); + const embeddingFixedMb = + headroomMb + supervisorMb + graphPoolMb + vectorPoolMb + modelMb + queueMb; + const includeParsing = request.includeParsing !== false; + const cpuWorkers = Math.max(1, cpus - 1); + const desiredWorkers = + request.workers === undefined + ? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers) + : integer(request.workers, 'parser workers'); + const explicitWorkerHeapMb = + request.workerHeapMb === undefined + ? undefined + : integer(request.workerHeapMb, 'worker heap reservation'); + const automaticWorkerHeapMb = + explicitWorkerHeapMb ?? + Math.min(4096, Math.max(MIN_WORKER_HEAP_MB, Math.floor(availableMb / 2 / desiredWorkers))); + const parseFixedMb = + headroomMb + + supervisorMb + + graphPoolMb + + (includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0); + let maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); + const mainHeapMb = + request.mainHeapMb === undefined + ? automatic + ? heapCapMbFor(request.effectiveMemoryBytes) + : Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35))) + : integer(request.mainHeapMb, 'main heap reservation'); + // Automatic sizing preserves the established independent V8 ceilings: the existing RAM-aware + // main heap, half RAM shared by parser heaps. Their finite caps + // can sum above RAM; they are not simultaneous resident allocations. Only an + // explicit --resource-budget imposes a sum ceiling. A heap override must not + // silently consume worker capacity or inflate the native pools/queues. + if (automatic) { + if ( + mainHeapMb > availableMb && + !request.allowHeapOvercommit && + request.mainHeapMb !== undefined + ) + throw new Error( + 'Resource budget cannot admit a main heap above effective memory without an explicit heap override', + ); + totalMb = Math.max( + totalMb, + mainHeapMb + embeddingFixedMb, + headroomMb + + supervisorMb + + graphPoolMb + + mainHeapMb + + (includeParsing ? desiredWorkers * (automaticWorkerHeapMb + WORKER_NATIVE_MB) : 0), + ); + maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb); + } + const totalBytes = bytes(totalMb, 'resource budget'); + // --memory-budget has an established 200 MiB minimum. Preserve that explicit + // choice; the larger floor is only the automatic sizing policy. + const minimumMainMb = request.mainHeapMb === undefined ? MIN_MAIN_HEAP_MB : 200; + if (mainHeapMb < minimumMainMb || mainHeapMb > maximumMainMb) { + const minimum = + Math.max(minimumMainMb, request.mainHeapMb ?? 0) + Math.max(embeddingFixedMb, parseFixedMb); + throw new Error( + `Resource budget ${totalMb} MiB cannot admit parser/model work; requires at least ${minimum} MiB for these allocations`, + ); + } + + // Explicit worker counts intentionally permit CPU oversubscription, as the + // parser's existing override does. Reserve every worker's heap/native memory + // below; automatic concurrency remains bounded by the usable CPU count. + const workerBudgetMb = totalMb - headroomMb - supervisorMb - graphPoolMb - mainHeapMb; + if ( + includeParsing && + explicitWorkerHeapMb !== undefined && + explicitWorkerHeapMb < MIN_WORKER_HEAP_MB + ) { + throw new Error(`Worker heap reservation must be at least ${MIN_WORKER_HEAP_MB} MiB`); + } + const workers = !includeParsing + ? 0 + : automatic + ? desiredWorkers + : request.workers === undefined + ? Math.min( + desiredWorkers, + Math.floor( + workerBudgetMb / ((explicitWorkerHeapMb ?? MIN_WORKER_HEAP_MB) + WORKER_NATIVE_MB), + ), + ) + : desiredWorkers; + const workerHeapMb = !includeParsing + ? 0 + : automatic + ? automaticWorkerHeapMb + : (explicitWorkerHeapMb ?? + Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB)); + const parseReservedMb = + headroomMb + + supervisorMb + + graphPoolMb + + mainHeapMb + + workers * (workerHeapMb + WORKER_NATIVE_MB); + if ( + (includeParsing && (workers < 1 || workerHeapMb < MIN_WORKER_HEAP_MB)) || + parseReservedMb > totalMb + ) { + throw new Error( + `Parser workers do not fit resource budget ${totalMb} MiB; reduce workers or explicit heap reservations`, + ); + } + const embeddingReservedMb = mainHeapMb + embeddingFixedMb; + return Object.freeze({ + totalBytes, + automatic, + headroomBytes: bytes(headroomMb, 'headroom'), + supervisorBytes: bytes(supervisorMb, 'supervisor reservation'), + mainHeapMb, + parse: Object.freeze({ + workers, + workerHeapMb, + workerNativeBytes: bytes(WORKER_NATIVE_MB, 'worker native reservation'), + graphPoolBytes, + nativeThreads: cpus, + reservedBytes: bytes(parseReservedMb, 'parse reservation'), + }), + embedding: Object.freeze({ + graphPoolBytes, + vectorPoolBytes: bytes(vectorPoolMb, 'vector pool'), + modelBytes: bytes(modelMb, 'model reservation'), + modelHeapMb: modelMb === 0 ? 0 : Math.min(1024, Math.floor(modelMb / 4)), + queueBytes: bytes(queueMb, 'embedding queues'), + nativeThreads: embeddingThreads, + reservedBytes: bytes(embeddingReservedMb, 'embedding reservation'), + }), + }); +} + +/** Account retained payloads before admission, including a prefetched unit. */ +export class EmbeddingByteBudget { + private retained = 0; + private peak = 0; + + constructor(readonly limit: number) { + integer(limit, 'embedding byte budget'); + } + + get usedBytes(): number { + return this.retained; + } + get peakBytes(): number { + return this.peak; + } + + reserve(size: number): () => void { + integer(size, 'retained payload bytes', 0); + if (size > this.limit - this.retained) { + throw new Error( + `Embedding payload requires ${size} bytes with ${this.limit - this.retained} bytes available`, + ); + } + this.retained += size; + this.peak = Math.max(this.peak, this.retained); + let released = false; + return () => { + if (released) throw new Error('Embedding byte reservation released twice'); + released = true; + this.retained -= size; + }; + } +} diff --git a/gitnexus/src/core/analysis-supervision.ts b/gitnexus/src/core/analysis-supervision.ts new file mode 100644 index 000000000..bfeca331d --- /dev/null +++ b/gitnexus/src/core/analysis-supervision.ts @@ -0,0 +1,7 @@ +/** Raised before native graph work when a CLI needs its phase supervisor. */ +export class AnalysisSupervisorRequired extends Error { + constructor() { + super('Graph work requires an analysis supervisor'); + this.name = 'AnalysisSupervisorRequired'; + } +} diff --git a/gitnexus/src/core/augmentation/engine.ts b/gitnexus/src/core/augmentation/engine.ts index ef6fe0c8c..8393990eb 100644 --- a/gitnexus/src/core/augmentation/engine.ts +++ b/gitnexus/src/core/augmentation/engine.ts @@ -21,7 +21,6 @@ import { requireRegisteredStoragePath, STATUS_STORAGE_REQUIREMENTS, } from '../../storage/storage-resolver.js'; -import { LBUG_DIRECTORY } from '../../storage/storage-constants.js'; import { BRANCHES_DIR, branchSlug } from '../../storage/branch-index.js'; import { getCurrentBranch } from '../../storage/git.js'; import { escapeCypherString } from '../lbug/cypher-escape.js'; @@ -85,10 +84,7 @@ async function findRepoForCwd(cwd: string): Promise<{ return { name: bestMatch.name, storagePath, - lbugPath: - indexDir === storagePath - ? resolveGraphPath(storagePath) - : path.join(indexDir, LBUG_DIRECTORY), + lbugPath: resolveGraphPath(indexDir), }; } catch { return null; diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 9216754ca..a2a78ad66 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -4,8 +4,18 @@ import { createRequire } from 'node:module'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; -import type { WorkerMessage } from '../../server/analyze-worker-protocol.js'; +import type { + WorkerMessage, + ContinueMessage, + GraphPreparedMessage, +} from '../../server/analyze-worker-protocol.js'; import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../analysis-host-resources.js'; +import { projectPreparedAnalysisHandoff } from '../analysis-phase-supervisor.js'; const _require = createRequire(import.meta.url); export type AutoSyncAnalysisRunner = ( @@ -20,6 +30,7 @@ export type AutoSyncAnalysisRunner = ( ) => Promise>; interface AnalysisWorker extends Pick { + pid?: number; stdout?: Pick | null; stderr?: Pick | null; unref?: () => void; @@ -95,8 +106,26 @@ export function createAutoSyncAnalysisRunner( overrides: Partial = {}, ): AutoSyncAnalysisRunner { const deps = { ...DEFAULT_DEPS, ...overrides }; - return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency, onProgress) => - new Promise>((resolve, reject) => { + return async ( + repoPath, + options, + timeoutMs, + signal, + onCancellationRequested, + concurrency, + onProgress, + ) => { + signal?.throwIfAborted(); + const resources = + options.resources ?? + resolveHostAnalysisResources({ + mainHeapMb: resolveWorkerHeapMb(concurrency) + 48, + includeEmbeddings: await analysisMayEmbed(repoPath, options), + supervisorBytes: Math.ceil(analysisSupervisorReservation() / (concurrency ?? 1)), + concurrency, + }); + const analysisOptions = { ...options, graphPublication: 'auto' as const, resources }; + return new Promise>((resolve, reject) => { if (signal?.aborted) { reject(new Error('Analysis cancelled.')); return; @@ -112,17 +141,16 @@ export function createAutoSyncAnalysisRunner( reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); return; } - const workerHeapMb = resolveWorkerHeapMb(concurrency); + const workerHeapMb = resources.mainHeapMb - 48; const execArgv = isDev ? [ '--import', pathToFileURL(_require.resolve('tsx/esm')).href, `--max-old-space-size=${workerHeapMb}`, + '--max-semi-space-size=16', ] - : [`--max-old-space-size=${workerHeapMb}`]; - const child = deps.forkWorker(workerPath, execArgv); - child.stdout?.resume(); - child.stderr?.resume(); + : [`--max-old-space-size=${workerHeapMb}`, '--max-semi-space-size=16']; + let child: AnalysisWorker; let terminalOutcome: WorkerMessage | undefined; let terminationError: Error | undefined; @@ -138,8 +166,8 @@ export function createAutoSyncAnalysisRunner( // so both handles have to go. Never a kill: the child may be inside // native work and is left to reach its own safe point. const releaseChild = () => { - child.channel?.unref?.(); - child.unref?.(); + child?.channel?.unref?.(); + child?.unref?.(); }; const settle = ( error?: Error, @@ -159,7 +187,7 @@ export function createAutoSyncAnalysisRunner( // IPC has the same semantics on macOS and Windows. The worker exits only // after reaching a JS-visible safe point; this parent keeps ownership until then. try { - child.send({ type: 'cancel' }); + child?.send({ type: 'cancel' }); } catch { // A closed IPC channel still has an exit/error path. Do not force-kill a // worker that may be inside native code. @@ -192,72 +220,122 @@ export function createAutoSyncAnalysisRunner( const onAbort = () => requestCancellation(new Error('Analysis cancelled.')); signal?.addEventListener('abort', onAbort, { once: true }); - child.on('message', (message: WorkerMessage) => { - // Once timeout/cancellation requested shutdown, its reason owns the - // result. A terminal IPC can already be queued behind cancellation. - if (terminalOutcome || terminationError) return; - if (message.type === 'progress') { - // The worker also sends its log lines as progress with percent -1; - // those are not progress. A listener's failure must not decide the - // run's outcome. - if (message.percent < 0) return; - try { - onProgress?.(message.percent, message.message); - } catch { - // ignored + const launch = (continuation?: ContinueMessage) => { + child = deps.forkWorker(workerPath, execArgv); + const phaseChild = child; + let prepared: GraphPreparedMessage | undefined; + child.stdout?.resume(); + child.stderr?.resume(); + child.on('message', (message: WorkerMessage) => { + // Once timeout/cancellation requested shutdown, its reason owns the + // result. A terminal IPC can already be queued behind cancellation. + if (terminalOutcome || terminationError || settled || child !== phaseChild) return; + if (message.type === 'progress') { + // Worker log messages use percent -1 and do not represent progress. + if (message.percent < 0) return; + try { + onProgress?.(message.percent, message.message); + } catch { + // A listener failure must not decide the run's outcome. + } + return; + } + if (message.type === 'graph-ready') { + // Watch has no serving reader cache. Publication is already verified. + try { + child.send({ type: 'graph-visible' }); + } catch (error) { + requestCancellation(error as Error); + } + return; + } + if (message.type === 'graph-prepared') { + try { + if (prepared || continuation) throw new Error('Repeated graph preparation'); + prepared = { ...message, handoff: projectPreparedAnalysisHandoff(message.handoff) }; + } catch (error) { + requestCancellation(error as Error); + } + return; + } + terminalOutcome = message; + deps.clearTimeoutFn(timeout); + }); + child.on('error', (error) => { + const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); + requestCancellation(workerError); + // This settles immediately rather than waiting out the grace, so the + // grace timer that would otherwise have released the child is cleared + // by cleanup(). Release it here instead — an errored channel does not + // mean the worker stopped. + releaseChild(); + settle(workerError); + }); + child.on('exit', (code, childSignal) => { + if (settled || child !== phaseChild) return; + if (terminationError) { + settle(terminationError); + return; + } + if (terminalOutcome?.type === 'complete') { + if (code === 0 && !childSignal) { + const { stats, graphWriteCollapsed } = terminalOutcome.result; + settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats }); + } else + settle( + new Error( + `Auto-sync analyze worker exited unsuccessfully (${childSignal ?? code}).`, + ), + ); + return; + } + if (!terminalOutcome && prepared && code === 0 && !childSignal && phaseChild.pid) { + try { + launch({ + type: 'continue', + repoPath, + options: analysisOptions, + writerPid: phaseChild.pid, + prepared, + }); + } catch (error) { + settle(error as Error); + } + return; + } + if (terminalOutcome?.type === 'error') { + settle( + new AutoSyncAnalysisError(terminalOutcome.message, { + code: terminalOutcome.code, + retryable: terminalOutcome.retryable, + }), + ); + return; } - return; - } - terminalOutcome = message; - deps.clearTimeoutFn(timeout); - }); - child.on('error', (error) => { - const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); - requestCancellation(workerError); - // This settles immediately rather than waiting out the grace, so the - // grace timer that would otherwise have released the child is cleared - // by cleanup(). Release it here instead — an errored channel does not - // mean the worker stopped. - releaseChild(); - settle(workerError); - }); - child.on('exit', (code, childSignal) => { - if (settled) return; - if (terminationError) { - settle(terminationError); - return; - } - if (terminalOutcome?.type === 'complete') { - const { stats, graphWriteCollapsed } = terminalOutcome.result; - // Only present on a collapsed write, so a healthy result keeps its shape. - settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats }); - return; - } - if (terminalOutcome?.type === 'error') { settle( - new AutoSyncAnalysisError(terminalOutcome.message, { - code: terminalOutcome.code, - retryable: terminalOutcome.retryable, - }), + new Error( + `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, + ), ); - return; + }); + try { + child.send(continuation ?? { type: 'start', repoPath, options: analysisOptions }); + } catch (error) { + const startError = new Error( + `Failed to start auto-sync analyze worker: ${(error as Error).message}`, + ); + requestCancellation(startError); + releaseChild(); + settle(startError); } - settle( - new Error( - `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, - ), - ); - }); + }; try { - child.send({ type: 'start', repoPath, options }); + launch(); } catch (error) { - const startError = new Error( - `Failed to start auto-sync analyze worker: ${(error as Error).message}`, - ); - requestCancellation(startError); - settle(startError); + settle(error as Error); } }); + }; } export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner(); diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts index 9306a8fd6..11315ef7d 100644 --- a/gitnexus/src/core/embeddings/ast-utils.ts +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -10,7 +10,6 @@ import { resolveLanguageKey, } from '../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; -import { getLanguageForFileContent, getProvider } from '../ingestion/languages/index.js'; import { extractNotebookPython, isNotebookPath } from '../ingestion/ipynb-extractor.js'; const parserCache = new Map(); @@ -20,6 +19,8 @@ const parserCache = new Map(); * Returns null if language is unavailable or parsing fails. */ export const ensureAndParse = async (content: string, filePath: string): Promise => { + const { getLanguageForFileContent, getProvider } = + await import('../ingestion/languages/index.js'); // Same classifier as ingest. Filename-only maps `.h` → C++, so Objective-C // headers (and method snippets from those headers) would parse with the // wrong grammar and miss class_interface / protocol_declaration / methods. diff --git a/gitnexus/src/core/embeddings/character-chunk.ts b/gitnexus/src/core/embeddings/character-chunk.ts index e4f6f949c..e6c3b7b0d 100644 --- a/gitnexus/src/core/embeddings/character-chunk.ts +++ b/gitnexus/src/core/embeddings/character-chunk.ts @@ -2,8 +2,6 @@ * Character-based sliding window chunking (pure, no tree-sitter dependency) */ -import { buildLineIndex, resolveChunkLines } from './line-index.js'; - export interface Chunk { text: string; chunkIndex: number; @@ -13,14 +11,71 @@ export interface Chunk { endLine: number; } +/** Sliding windows with constant auxiliary memory, including line tracking. */ +export function* iterateCharacterChunks( + content: string, + startLine: number, + endLine: number, + chunkSize = 1200, + overlap = 120, +): Generator { + if ( + !Number.isSafeInteger(chunkSize) || + chunkSize < 1 || + !Number.isSafeInteger(overlap) || + overlap < 0 + ) + throw new Error('Invalid character chunk size or overlap'); + if (content.length <= chunkSize) { + yield { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }; + return; + } + let offset = 0; + let chunkIndex = 0; + let relativeStart = 0; + let relativeEnd = 0; + let nextStartNewline = content.indexOf('\n'); + let nextEndNewline = nextStartNewline; + while (offset < content.length) { + const end = Math.min(offset + chunkSize, content.length); + while (nextStartNewline >= 0 && nextStartNewline < offset) { + relativeStart++; + nextStartNewline = content.indexOf('\n', nextStartNewline + 1); + } + while (nextEndNewline >= 0 && nextEndNewline < end - 1) { + relativeEnd++; + nextEndNewline = content.indexOf('\n', nextEndNewline + 1); + } + yield { + text: content.slice(offset, end), + chunkIndex: chunkIndex++, + startOffset: offset, + endOffset: end, + startLine: startLine + relativeStart, + endLine: startLine + relativeEnd, + }; + offset = end - overlap; + if (offset >= content.length || end >= content.length) break; + if (offset <= (chunkIndex > 1 ? end - chunkSize : 0)) offset = end; + } +} + +/** Compatibility API for callers that need all chunks. */ export const characterChunk = ( content: string, startLine: number, endLine: number, - chunkSize: number = 1200, - overlap: number = 120, + chunkSize = 1200, + overlap = 120, ): Chunk[] => { - if (content.length <= chunkSize) { + if (content.length <= chunkSize) return [ { text: content, @@ -31,33 +86,5 @@ export const characterChunk = ( endLine, }, ]; - } - - const chunks: Chunk[] = []; - let offset = 0; - const lineOffsets = buildLineIndex(content); - - while (offset < content.length) { - const end = Math.min(offset + chunkSize, content.length); - const chunkText = content.slice(offset, end); - const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine); - - chunks.push({ - text: chunkText, - chunkIndex: chunks.length, - startOffset: offset, - endOffset: end, - startLine: lineRange.startLine, - endLine: lineRange.endLine, - }); - - offset = end - overlap; - if (offset >= content.length) break; - if (end >= content.length) break; - if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) { - offset = end; - } - } - - return chunks; + return [...iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap)]; }; diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts index f3e64aaea..6dfba2a9c 100644 --- a/gitnexus/src/core/embeddings/chunker.ts +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -9,7 +9,7 @@ export { type Chunk, characterChunk } from './character-chunk.js'; -import { characterChunk } from './character-chunk.js'; +import { iterateCharacterChunks } from './character-chunk.js'; import type { Chunk } from './character-chunk.js'; import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js'; import { buildLineIndex, resolveChunkLines } from './line-index.js'; @@ -81,10 +81,68 @@ export const chunkNode = async ( filePath: string, startLine: number, endLine: number, - chunkSize: number = 1200, - overlap: number = 120, + chunkSize = 1200, + overlap = 120, options?: ChunkOptions, ): Promise => { + if (label !== 'Class' && content.length <= chunkSize) + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + return [ + ...(await prepareNodeChunks( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + options, + )), + ]; +}; + +/** Produce one chunk at a time, without retaining generated text for the node. */ +export async function* iterateNodeChunks( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize = 1200, + overlap = 120, + options?: ChunkOptions, +): AsyncGenerator { + yield* await prepareNodeChunks( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + options, + ); +} + +async function prepareNodeChunks( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, + options?: ChunkOptions, +): Promise> { // Even a short class contains implementations. Never fall back to raw Class // source: an unavailable parser yields identity/name metadata only. if (label === 'Class') { @@ -105,28 +163,17 @@ export const chunkNode = async ( return [classMetadataChunk(startLine, endLine)]; } } - // Content fits in one chunk — no splitting needed if (content.length <= chunkSize) { - return [ - { - text: content, - chunkIndex: 0, - startOffset: 0, - endOffset: content.length, - startLine, - endLine, - }, - ]; + return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); } - const rule = CHUNKING_RULES[label]; if (!rule || options?.strategy === 'fixed') { - return characterChunk(content, startLine, endLine, chunkSize, overlap); + return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); } - + let chunks: Generator | undefined; try { - if (rule.mode === CHUNK_MODE_AST_FUNCTION) { - const astChunks = await astChunk( + if (rule?.mode === CHUNK_MODE_AST_FUNCTION) { + chunks = await astChunk( content, filePath, startLine, @@ -136,11 +183,8 @@ export const chunkNode = async ( rule, options?.name, ); - if (astChunks.length > 0) return astChunks; - } - - if (rule.mode === CHUNK_MODE_AST_DECLARATION) { - const declarationChunks = await declarationChunk( + } else if (rule?.mode === CHUNK_MODE_AST_DECLARATION) { + chunks = await declarationChunk( content, filePath, startLine, @@ -149,15 +193,13 @@ export const chunkNode = async ( overlap, rule, ); - if (declarationChunks.length > 0) return declarationChunks; } } catch { - // AST parsing failed — fall through to character fallback + // Only preparation may fall back. Never restart after yielding a chunk: + // the consumer may already have durably stored it. } - - // Character-based fallback for everything else - return characterChunk(content, startLine, endLine, chunkSize, overlap); -}; + return chunks ?? iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap); +} /** * AST-based chunking for Function/Method @@ -173,9 +215,9 @@ const astChunk = async ( overlap: number, rule: ChunkingRule, expectedName?: string, -): Promise => { +): Promise | undefined> => { const tree = await ensureAndParse(content, filePath); - if (!tree) return []; + if (!tree) return undefined; const root = tree.rootNode; const lineOffsets = buildLineIndex(content); @@ -183,11 +225,11 @@ const astChunk = async ( // Find the function/method declaration in the snippet AST. // tree-sitter parses node.content (a snippet), so rows are relative (0-based). const targetNode = findFunctionNode(root, expectedName); - if (!targetNode) return []; + if (!targetNode) return undefined; // Get the body (statements) via childForFieldName('body') const bodyNode = targetNode.childForFieldName('body'); - if (!bodyNode) return []; + if (!bodyNode) return undefined; // Extract individual statements const statements: Array<{ startIndex: number; endIndex: number }> = []; @@ -208,7 +250,7 @@ const astChunk = async ( } } - if (statements.length === 0) return []; + if (statements.length === 0) return undefined; // A fitting statement must not fall into character splitting just because // its function header or closing delimiters consume the remaining budget. @@ -397,18 +439,18 @@ const declarationChunk = async ( chunkSize: number, overlap: number, rule: ChunkingRule, -): Promise => { +): Promise | undefined> => { const tree = await ensureAndParse(content, filePath); - if (!tree) return []; + if (!tree) return undefined; const targetNode = findDeclarationNode(tree.rootNode); - if (!targetNode) return []; + if (!targetNode) return undefined; const bodyNode = getDeclarationBodyNode(targetNode); - if (!bodyNode) return []; + if (!bodyNode) return undefined; const members = collectDeclarationUnits(bodyNode, rule.groupFields); - if (members.length === 0) return []; + if (members.length === 0) return undefined; return chunkByUnits( content, @@ -443,7 +485,7 @@ const buildChunk = ( }; }; -const chunkByUnits = ( +function* chunkByUnits( content: string, lineOffsets: Int32Array, baseStartLine: number, @@ -454,8 +496,8 @@ const chunkByUnits = ( containerEndOffset: number, includeContainerPrefixOnFirstChunk: boolean, includeContainerSuffixOnLastChunk: boolean, -): Chunk[] => { - const chunks: Chunk[] = []; +): Generator { + let chunkIndex = 0; let chunkStartUnitIdx = 0; while (chunkStartUnitIdx < units.length) { @@ -496,32 +538,31 @@ const chunkByUnits = ( oversizedEndOffset, baseStartLine, ); - const oversizedChunks = characterChunk( + for (const chunk of iterateCharacterChunks( content.slice(oversizedStartOffset, oversizedEndOffset), oversizedLineRange.startLine, oversizedLineRange.endLine, chunkSize, overlap, - ).map((chunk, offsetIdx) => ({ - ...chunk, - chunkIndex: chunks.length + offsetIdx, - startOffset: chunk.startOffset + oversizedStartOffset, - endOffset: chunk.endOffset + oversizedStartOffset, - })); - chunks.push(...oversizedChunks); + )) { + yield { + ...chunk, + chunkIndex: chunkIndex++, + startOffset: chunk.startOffset + oversizedStartOffset, + endOffset: chunk.endOffset + oversizedStartOffset, + }; + } chunkStartUnitIdx += 1; continue; } - chunks.push( - buildChunk( - content, - lineOffsets, - chunks.length, - chunkStartOffset, - candidateEndOffset, - baseStartLine, - ), + yield buildChunk( + content, + lineOffsets, + chunkIndex++, + chunkStartOffset, + candidateEndOffset, + baseStartLine, ); if (chunkEndUnitIdx === units.length - 1) { @@ -540,9 +581,7 @@ const chunkByUnits = ( chunkStartUnitIdx = nextChunkStartUnitIdx; } } - - return chunks; -}; +} const findOverlapStartIndex = ( statements: Array<{ startIndex: number; endIndex: number }>, @@ -654,7 +693,7 @@ const projectClassChunks = ( overlap: number, strategy: EmbeddingConfig['chunkStrategy'] = 'ast', expectedName?: string, -): EmbeddingChunk[] => { +): Iterable => { const target = tree && findDeclarationNode(tree.rootNode, expectedName); const body = target && getDeclarationBodyNode(target); if (!body || target.hasError) return [classMetadataChunk(startLine, endLine)]; @@ -745,7 +784,7 @@ const projectClassChunks = ( const projected = pieces.join(''); const chunks = strategy === 'fixed' - ? characterChunk(projected, 1, 1, chunkSize, overlap) + ? iterateCharacterChunks(projected, 1, 1, chunkSize, overlap) : chunkByUnits( projected, buildLineIndex(projected), @@ -772,17 +811,19 @@ const projectClassChunks = ( segment.source + Math.max(0, Math.min(offset - segment.start, segment.end - segment.start)) ); }; - return chunks.map((chunk) => { - const startOffset = sourceOffset(chunk.startOffset); - const endOffset = sourceOffset(chunk.endOffset - 1) + 1; - return { - ...chunk, - startOffset, - endOffset, - ...resolveChunkLines(originalLines, startOffset, endOffset, startLine), - classDeclaration: header, - }; - }); + return (function* () { + for (const chunk of chunks) { + const startOffset = sourceOffset(chunk.startOffset); + const endOffset = sourceOffset(chunk.endOffset - 1) + 1; + yield { + ...chunk, + startOffset, + endOffset, + ...resolveChunkLines(originalLines, startOffset, endOffset, startLine), + classDeclaration: header, + }; + } + })(); }; const collectDeclarationUnits = ( diff --git a/gitnexus/src/core/embeddings/embedding-inputs.ts b/gitnexus/src/core/embeddings/embedding-inputs.ts new file mode 100644 index 000000000..3e81667c9 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-inputs.ts @@ -0,0 +1,129 @@ +/** Shared streaming input layout for generation and historical-vector validation. */ +import { isNotebookPath } from '../ingestion/ipynb-extractor.js'; +import { ensureAndParse } from './ast-utils.js'; +import { iterateNodeChunks, type EmbeddingChunk } from './chunker.js'; +import { extractStructuralNames } from './structural-extractor.js'; +import { + generateEmbeddingText, + splitEmbeddingText, + withoutOverlappingContext, +} from './text-generator.js'; +import { needsEmbeddingSource, type EmbeddingSourceContext } from './source-reader.js'; +import { + isShortLabel, + STRUCTURAL_LABELS, + type EmbeddableNode, + type EmbeddingConfig, +} from './types.js'; + +export interface EmbeddingInput { + text: string; + startLine: number; + endLine: number; +} + +export interface EmbeddingReuseOptions { + node: EmbeddableNode; + prepareInputValidation?: () => Promise; + inputs?: () => AsyncIterable; + matchesHash?: (stored: string) => Promise; +} + +export async function* iterateEmbeddingInputs( + indexedNode: EmbeddableNode, + config: EmbeddingConfig, + options: { + source?: EmbeddingSourceContext; + readSource?: (node: EmbeddableNode) => Promise<{ content: string; release: () => void }>; + signal?: AbortSignal; + } = {}, +): AsyncGenerator { + const hydrate = + options.source?.contentRetention === 'full' && needsEmbeddingSource(indexedNode, config); + if (hydrate && !options.readSource) + throw new Error('Original embedding source needs a budgeted reader'); + const lease = hydrate ? await options.readSource?.(indexedNode) : undefined; + // Never retain expanded source or structural metadata in the fetched unit. + const node = { ...indexedNode, content: lease?.content ?? indexedNode.content }; + try { + options.signal?.throwIfAborted(); + let classTree: Awaited> | undefined; + if (node.label === 'Class') { + try { + classTree = await ensureAndParse(node.content, node.filePath); + } catch { + classTree = null; + } + } + if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames( + node.content, + node.filePath, + classTree, + node.label === 'Class' ? node.name : undefined, + ); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + /* Structural enrichment remains optional. */ + } + } + const startLine = node.startLine ?? 0, + endLine = node.endLine ?? 0; + const chunks: Iterable | AsyncIterable = isShortLabel( + node.label, + ) + ? [ + { + text: node.content, + chunkIndex: 0, + startLine, + endLine, + startOffset: 0, + endOffset: node.content.length, + }, + ] + : iterateNodeChunks( + node.label, + node.content, + node.filePath, + startLine, + endLine, + config.chunkSize, + config.overlap, + { strategy: config.chunkStrategy, name: node.name, tree: classTree }, + ); + const conservativeRange = + options.source && + (options.source.contentRetention !== 'full' || isNotebookPath(node.filePath)); + let prevTail = '', + previousEndOffset: number | undefined; + for await (const chunk of chunks) { + options.signal?.throwIfAborted(); + node.classDeclaration = chunk.classDeclaration; + const text = generateEmbeddingText( + node, + chunk.text, + config, + chunk.chunkIndex, + withoutOverlappingContext(prevTail, previousEndOffset, chunk.startOffset), + ); + for (const part of splitEmbeddingText(text, config.maxInputChars)) { + // Detach slices so queued inputs cannot pin a large declaration/source. + yield { + text: + lease || text.length > config.maxInputChars + ? Buffer.from(part, 'utf16le').toString('utf16le') + : part, + startLine: conservativeRange ? startLine : chunk.startLine, + endLine: conservativeRange ? endLine : chunk.endLine, + }; + } + prevTail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + previousEndOffset = chunk.endOffset; + } + } finally { + lease?.release(); + } +} diff --git a/gitnexus/src/core/embeddings/embedding-job-identity.ts b/gitnexus/src/core/embeddings/embedding-job-identity.ts new file mode 100644 index 000000000..c55d4930d --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job-identity.ts @@ -0,0 +1,80 @@ +import { createHash } from 'node:crypto'; +import type { EmbeddingIdentity } from './embedding-identity.js'; +import type { EmbeddingConfig } from './types.js'; +import type { ContentRetention } from '../../storage/repo-meta.js'; + +export interface EmbeddingGraphIdentity { + generation: string; + coveredContent: string; + revision: string; + analyzer: string; + parser: string; + schema: string; + vectorSchema: string; +} + +const digest = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); + +/** Explicit fields exclude credentials and scheduling settings from persisted identity. */ +export function embeddingJobIdentity( + graph: EmbeddingGraphIdentity, + vector: EmbeddingIdentity, + textVersion: string, + config: Pick< + EmbeddingConfig, + 'chunkSize' | 'overlap' | 'maxSnippetLength' | 'maxDescriptionLength' | 'maxInputChars' + > & + Partial>, + contentRetention: ContentRetention = 'full', +): { identity: string; sourceIdentity: string; vectorIdentity: string; inputIdentity: string } { + const source = [ + graph.generation, + graph.coveredContent, + graph.revision, + graph.analyzer, + graph.parser, + graph.schema, + graph.vectorSchema, + ]; + if ( + source.some((value) => typeof value !== 'string' || !value) || + !vector.model || + !/^(local|http:[a-f0-9]{64})$/.test(vector.provider) || + !Number.isSafeInteger(vector.dimensions) || + vector.dimensions < 1 || + !textVersion || + ![ + config.chunkSize, + config.overlap, + config.maxSnippetLength, + config.maxDescriptionLength, + config.maxInputChars, + ].every((n) => Number.isSafeInteger(n) && n >= 0) || + config.chunkSize < 1 || + config.overlap >= config.chunkSize || + config.maxInputChars < 2 || + !['ast', 'fixed'].includes(config.chunkStrategy ?? 'ast') || + !['full', 'symbol', 'none'].includes(contentRetention) + ) + throw new Error('Incomplete embedding job identity'); + const sourceIdentity = digest(source); + const vectorIdentity = digest([vector.provider, vector.model, vector.dimensions]); + const inputIdentity = digest([ + 'source-aware-ast-inputs:v1', + textVersion, + config.chunkStrategy ?? 'ast', + contentRetention, + config.chunkSize, + config.overlap, + config.maxSnippetLength, + config.maxDescriptionLength, + config.maxInputChars, + ]); + return { + sourceIdentity, + vectorIdentity, + inputIdentity, + identity: digest([1, sourceIdentity, vectorIdentity, inputIdentity]), + }; +} diff --git a/gitnexus/src/core/embeddings/embedding-job-runner.ts b/gitnexus/src/core/embeddings/embedding-job-runner.ts new file mode 100644 index 000000000..6206529c9 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job-runner.ts @@ -0,0 +1,604 @@ +/** One durable manifest unit, one bounded provider batch, no corpus-sized maps. */ +import { createHash } from 'node:crypto'; +import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; +import type { VectorNodeReceipt, VectorStore, VectorWriteChunk } from '../lbug/vector-store.js'; +import { + embeddingInputFailureKey, + type EmbeddingJobJournal, + type EmbeddingJobState, + type EmbeddingUnitIntent, +} from '../../storage/embedding-job-journal.js'; +import { + fetchEmbeddingWorkUnit, + readEmbeddingWorkUnits, + type EmbeddingManifestReceipt, + type EmbeddingManifestSource, + type EmbeddingWorkUnit, +} from './embedding-work-manifest.js'; +import { contentHashForNode, embedInputRanges, isInputRejection } from './embedding-pipeline.js'; +import { iterateEmbeddingInputs, type EmbeddingReuseOptions } from './embedding-inputs.js'; +import { + EmbeddingSourceReader, + needsEmbeddingSource, + type EmbeddingSourceContext, +} from './source-reader.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { type EmbeddingConfig, type EmbeddableNode } from './types.js'; +import type { embedBatch } from './embedder.js'; + +export interface EmbeddingReuseSource { + options: { batchBytes: number }; + requiresInputValidation?: boolean; + reusable( + nodeId: string, + contentHash: string, + options: EmbeddingReuseOptions, + ): Promise; + readChunks: VectorStore['readChunks']; +} + +export interface EmbeddingJobRunnerOptions { + directory: string; + manifest: EmbeddingManifestReceipt; + source: EmbeddingManifestSource; + sourceContext?: EmbeddingSourceContext; + journal: EmbeddingJobJournal; + store: VectorStore; + /** The controller has validated vector/input identities before providing reuse. */ + reuse?: EmbeddingReuseSource; + /** The writer was adopted from an older graph. Persist this choice for the + * whole job: old complete receipts cannot skip current-input validation. */ + revalidateStoredInputs?: boolean; + config?: Partial; + queueBytes: number; + provider: typeof embedBatch; + initializeProvider?: () => Promise; + signal?: AbortSignal; + onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void; +} + +export interface EmbeddingJobRunResult { + state: EmbeddingJobState; + peakQueueBytes: number; + /** Absent while failed work remains. Publication is the controller's job. */ + indexKind?: 'hnsw' | 'exact'; +} + +/** Caller owns the job lock and closes all handles, including on rejection. */ +export async function runEmbeddingJob( + options: EmbeddingJobRunnerOptions, +): Promise { + const { directory, manifest, source, journal, store, signal } = options; + const config = resolveEmbeddingConfig(options.config ?? {}); + const budget = new EmbeddingByteBudget(options.queueBytes); + let sourceReader: EmbeddingSourceReader | undefined; + let releaseSourceCache: (() => void) | undefined; + const dimensions = store.options.dimensions; + // UTF-16 text + worst-case JSON escaping/encoding, bounded HTTP response + // decoding/JSON parsing, float results, JS arrays and native copies. HTTP + // responses are capped at 4096 + 32*dimensions bytes per row by the controller. + // The provider's runtime has a separate reservation. + const rowBytes = (textChars: number, idChars: number) => + 16 * textChars + 512 * dimensions + 64 * 1024 + 4 * idChars; + const minimumBatchBytes = rowBytes(config.maxInputChars, 0); + if (minimumBatchBytes > budget.limit) + throw new Error('Embedding input/vector batch does not fit the resource budget'); + let initialized = false; + let attempted = 0; + let failures = 0; + let consecutiveFailures = 0; + const remainingNodes = + manifest.totalNodes - journal.snapshot().completedNodes - journal.snapshot().reusedNodes; + const ratioFloor = Math.min(20, Math.max(5, Math.ceil(remainingNodes / config.subBatchSize / 2))); + // Bounded even if every source node is rejected. IDs are needed only until + // the existing five-node input-rejection guard fires or a success resets it. + const rejected = new Set(); + const progress = () => options.onProgress?.(journal.snapshot(), budget.peakBytes); + + const processUnit = async ( + unit: EmbeddingWorkUnit, + intent: EmbeddingUnitIntent, + ): Promise => { + signal?.throwIfAborted(); + const active = journal.snapshot().active ?? intent; + const history = active.inputFailures ?? {}; + const excluded = new Set(); + if (Object.keys(history).length) { + const nodesByKey = new Map( + unit.nodes.map((node) => [ + active.inputFailureKeyFormat === 'sha256' ? embeddingInputFailureKey(node.id) : node.id, + node.id, + ]), + ); + for (const [key, attempts] of Object.entries(history)) { + const id = nodesByKey.get(key); + if (id === undefined) + throw new Error('Embedding rejection history does not match its manifest unit'); + if (attempts >= 3) excluded.add(id); + } + } + const complete = options.revalidateStoredInputs + ? new Map() + : await store.completedNodes(unit.nodes.map((node) => node.id)); + for (const id of excluded) complete.delete(id); + const missing = unit.nodes.filter((node) => !complete.has(node.id) && !excluded.has(node.id)); + const retainedBytes = missing.reduce( + (n, key) => n + 2 * (key.contentBytes + key.metadataBytes), + 0, + ); + // One node may expand its text/AST while the rest of the fetched unit stays + // retained. Admission precedes intent, provider calls and vector mutation. + const scratchBytes = missing.reduce( + (n, key) => Math.max(n, 16 * (key.contentBytes + key.metadataBytes)), + 0, + ); + const maxIdChars = missing.reduce((n, key) => Math.max(n, key.id.length), 0); + const maximumBatchRows = Math.min( + 256, + config.subBatchSize, + Math.floor(store.options.batchBytes / (16 * dimensions + 128 + 2 * (maxIdChars + 40))), + ); + if (maximumBatchRows < 1) throw new Error('Vector write does not fit its batch reservation'); + const minimumUnitBatchBytes = rowBytes(config.maxInputChars, maxIdChars); + const unitBytes = retainedBytes + scratchBytes + minimumUnitBatchBytes; + if (releaseSourceCache && budget.usedBytes + unitBytes > budget.limit) { + sourceReader = undefined; + releaseSourceCache(); + releaseSourceCache = undefined; + } + if (budget.usedBytes + unitBytes > budget.limit) + throw new Error( + `Embedding unit ${unit.ordinal} requires a larger resource budget; no source text was truncated`, + ); + if (!journal.snapshot().active) await journal.begin(intent); + if (excluded.size) await store.excludeNodes([...excluded]); + progress(); + if (!missing.length) { + if (complete.size) await store.checkpointUnit([...complete.values()]); + await journal.acknowledge({ + reusedNodes: complete.size, + chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), + }); + progress(); + return; + } + const fetched = await fetchEmbeddingWorkUnit( + source, + { ...unit, nodes: missing, retainedBytes }, + budget, + ); + const releaseScratch = budget.reserve(scratchBytes); + let batch: Array & { text: string; release: () => void }> = + []; + // Retained vector copies share one bounded write buffer, whether their + // source is a reusable artifact or a newly completed provider request. + let copyBatch: VectorWriteChunk[] = []; + let copyBytes = 0; + let copyReleases: Array<() => void> = []; + const generated = new Map< + string, + { contentHash: string; chunks: number; ended: boolean; inputsDigest?: string } + >(); + const failed = new Set(); + const rejectedInputs = new Set(); + let failureCause: 'input' | 'provider' = 'input'; + let abort: unknown; + let reusedNodes = complete.size; + + const flushCopies = async (): Promise => { + if (!copyBatch.length) return; + signal?.throwIfAborted(); + try { + await store.appendBatch(copyBatch); + } finally { + for (const release of copyReleases) release(); + copyBatch = []; + copyBytes = 0; + copyReleases = []; + } + }; + + const queueWrites = async (chunks: VectorWriteChunk[]): Promise => { + const bytes = chunks.reduce( + (total, chunk) => total + 32 * dimensions + 1024 + 4 * chunk.nodeId.length, + 0, + ); + if ( + copyBatch.length && + (copyBatch.length + chunks.length > 256 || + copyBytes + bytes > store.options.batchBytes || + budget.usedBytes + bytes > budget.limit) + ) + await flushCopies(); + // The active provider batch already reserves these vector copies. A + // tight queue can write immediately without a second retained reservation. + if (bytes > store.options.batchBytes || budget.usedBytes + bytes > budget.limit) { + await store.appendBatch(chunks); + return; + } + const release = budget.reserve(bytes); + copyBatch.push(...chunks); + copyBytes += bytes; + copyReleases.push(release); + }; + + const flush = async (): Promise => { + if (!batch.length) return; + signal?.throwIfAborted(); + if (!initialized) { + await options.initializeProvider?.(); + initialized = true; + } + attempted++; + let providerFailed = false; + try { + for await (const result of embedInputRanges( + batch.map((row) => row.text), + batch.length, + signal, + options.provider, + )) { + signal?.throwIfAborted(); + const rows = batch.slice(result.start, result.end); + if ('error' in result) { + for (const row of rows) failed.add(row.nodeId); + if (isInputRejection(result.error)) { + for (const row of rows) rejectedInputs.add(row.nodeId); + for (const row of rows) if (rejected.size < 5) rejected.add(row.nodeId); + } else { + failureCause = 'provider'; + providerFailed = true; + consecutiveFailures++; + if (consecutiveFailures >= 5) abort = result.error; + } + continue; + } + if ( + result.embeddings.length !== rows.length || + result.embeddings.some( + (v) => v.length !== dimensions || v.some((n) => !Number.isFinite(n)), + ) + ) + throw new Error('Embedding provider returned invalid vector coverage'); + consecutiveFailures = 0; + rejected.clear(); + const updates = rows.flatMap((row, i) => + failed.has(row.nodeId) + ? [] + : [ + { + nodeId: row.nodeId, + contentHash: row.contentHash, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + embedding: Array.from(result.embeddings[i]), + }, + ], + ); + if (updates.length) await queueWrites(updates); + } + if (providerFailed) failures++; + if (!abort && providerFailed && attempted >= ratioFloor && failures / attempted >= 0.25) + abort = new Error( + 'Embedding endpoint failure rate exceeded 25%; unfinished units remain resumable', + ); + if (!abort && rejected.size >= 5) + abort = new Error( + 'Embedding inputs rejected for five nodes without a success; unfinished units remain resumable', + ); + } finally { + for (const row of batch) row.release(); + batch = []; + } + }; + + const prepareInputValidation = options.reuse?.requiresInputValidation + ? async () => { + // Release a provider batch only when a matching legacy group needs + // input expansion. Ineligible groups keep normal provider batching. + await flush(); + if (abort) throw abort; + } + : undefined; + + const nodeInputs = (node: EmbeddableNode) => + iterateEmbeddingInputs(node, config, { + source: options.sourceContext, + signal, + readSource: async (indexedNode) => { + const sourceContext = options.sourceContext; + if (!sourceContext) throw new Error('Missing indexed embedding source context'); + let release: (() => void) | undefined; + try { + if (!sourceReader) { + const cacheBytes = Math.min(16 * 1024 * 1024, Math.floor(budget.limit / 8)); + if (budget.usedBytes + cacheBytes + minimumUnitBatchBytes > budget.limit) { + await flush(); + await flushCopies(); + } + if (abort) throw abort; + releaseSourceCache = budget.reserve(cacheBytes); + sourceReader = new EmbeddingSourceReader(sourceContext, cacheBytes); + } + const content = await sourceReader.read(indexedNode, async (fileBytes) => { + signal?.throwIfAborted(); + // Whole-file decoding/line index, notebook reconstruction, AST and + // projected declarations live until this node's iterator closes. + const bytes = 64 * 1024 + 16 * fileBytes; + if (budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit) { + await flush(); + await flushCopies(); + } + if (abort) throw abort; + if ( + !Number.isSafeInteger(bytes) || + budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit + ) + throw new Error( + 'Original embedding source requires a larger resource budget; no source text was truncated', + ); + release = budget.reserve(bytes); + }); + if (!release) throw new Error('Original embedding source was not admitted'); + return { content, release }; + } catch (error) { + release?.(); + throw error; + } + }, + }); + + const sourceAware = (node: EmbeddableNode) => + options.sourceContext?.contentRetention === 'full' && needsEmbeddingSource(node, config); + const inputHasher = () => + createHash('sha1').update('embedding-inputs:v1\n').update(JSON.stringify(config.modelId)); + + try { + // Hash once per fetched node; current-job complete groups never fetch/hash. + const inputs = fetched.nodes.map((node) => ({ + nodeId: node.id, + contentHash: contentHashForNode(node, config, options.sourceContext), + sourceStartLine: node.startLine ?? 0, + sourceEndLine: node.endLine ?? 0, + })); + const reusableNodes = new Map(); + for (let i = 0; i < inputs.length; i++) { + const input = inputs[i], + node = fetched.nodes[i]; + const baseHash = input.contentHash; + let digest: string | undefined; + const matchesHash = async (previous: string): Promise => { + if (!sourceAware(node) || !/^[a-f0-9]{40}:[a-f0-9]{40}$/.test(previous)) return false; + if (previous.startsWith(`${baseHash}:`)) { + input.contentHash = previous; + return true; + } + if (!digest) { + const hash = inputHasher(); + for await (const part of nodeInputs(node)) + hash.update(JSON.stringify([part.text, part.startLine, part.endLine])); + digest = hash.digest('hex'); + } + if (!previous.endsWith(`:${digest}`)) return false; + input.contentHash = `${baseHash}:${digest}`; + return true; + }; + if (options.revalidateStoredInputs) { + const retained = await store.reuseNode(input, async (previous) => + (await matchesHash(previous)) ? input.contentHash : undefined, + ); + if (retained) { + complete.set(input.nodeId, retained); + reusedNodes++; + continue; + } + } + const reusable = await options.reuse?.reusable(node.id, input.contentHash, { + node, + prepareInputValidation, + inputs: () => nodeInputs(node), + matchesHash, + }); + if (reusable) reusableNodes.set(node.id, reusable); + else input.contentHash = baseHash; + } + const missingInputs = options.revalidateStoredInputs + ? inputs.filter((input) => !complete.has(input.nodeId)) + : inputs; + if (missingInputs.length) await store.beginNodes(missingInputs); + const copied: Array<{ nodeId: string; contentHash: string; chunks: number }> = []; + for (let i = 0; i < fetched.nodes.length; i++) { + const node = fetched.nodes[i]; + const input = inputs[i]; + signal?.throwIfAborted(); + if (options.revalidateStoredInputs && complete.has(node.id)) continue; + const reuse = options.reuse; + const reusable = reusableNodes.get(node.id); + if (reusable && reuse) { + // Flush before copy so there is only one vector batch in memory. + await flush(); + if (abort) break; + const copyRowBytes = 32 * dimensions + 1024 + 4 * node.id.length; + for (let start = 0; start < reusable.chunks; ) { + if ( + copyBatch.length && + (copyBatch.length === 256 || + Math.min(store.options.batchBytes - copyBytes, budget.limit - budget.usedBytes) < + copyRowBytes) + ) + await flushCopies(); + const copyRows = Math.min( + 256 - copyBatch.length, + Math.floor( + Math.min( + reuse.options.batchBytes, + store.options.batchBytes - copyBytes, + budget.limit - budget.usedBytes, + ) / copyRowBytes, + ), + ); + if (copyRows < 1) + throw new Error('Reusable vector batch does not fit the resource budget'); + const length = Math.min(copyRows, reusable.chunks - start); + // Reuse has no HTTP body or JSON decoder; retain only vector copies. + const bytes = length * copyRowBytes; + const release = budget.reserve(bytes); + try { + const chunks = await reuse.readChunks(reusable, start, length); + const delta = input.sourceStartLine - (reusable.sourceStartLine ?? 0); + copyBatch.push( + ...chunks.map((chunk) => ({ + ...chunk, + nodeId: node.id, + contentHash: input.contentHash, + startLine: chunk.startLine + delta, + endLine: + chunk.endLine === reusable.sourceEndLine + ? input.sourceEndLine + : chunk.endLine + delta, + })), + ); + } catch (error) { + release(); + throw error; + } + copyReleases.push(release); + copyBytes += bytes; + start += length; + } + copied.push({ nodeId: node.id, contentHash: input.contentHash, chunks: reusable.chunks }); + reusedNodes++; + continue; + } + const state = { + contentHash: input.contentHash, + chunks: 0, + ended: false, + inputsDigest: undefined as string | undefined, + }; + generated.set(node.id, state); + const hash = sourceAware(node) ? inputHasher() : undefined; + for await (const generatedInput of nodeInputs(node)) { + if (failed.has(node.id) || abort) break; + hash?.update( + JSON.stringify([generatedInput.text, generatedInput.startLine, generatedInput.endLine]), + ); + const bytes = rowBytes(generatedInput.text.length, node.id.length); + if ( + batch.length && + (batch.length >= maximumBatchRows || budget.usedBytes + bytes > budget.limit) + ) + await flush(); + if (failed.has(node.id) || abort) break; + if (copyBatch.length && budget.usedBytes + bytes > budget.limit) await flushCopies(); + const release = budget.reserve(bytes); + batch.push({ + ...input, + chunkIndex: state.chunks++, + ...generatedInput, + release, + }); + } + state.ended = !failed.has(node.id) && !abort; + if (state.ended) state.inputsDigest = hash?.digest('hex'); + if (abort) break; + } + if (!abort) await flush(); + if (copyBatch.length) await flushCopies(); + const attempts = await journal.recordInputRejections([...rejectedInputs]); + for (const id of rejectedInputs) { + if (attempts[id] < 3) continue; + await store.excludeNodes([id]); + excluded.add(id); + generated.delete(id); + failed.delete(id); + } + const finished = [ + ...[...generated] + .filter(([id, node]) => node.ended && !failed.has(id)) + .map(([nodeId, node]) => ({ + nodeId, + contentHash: node.contentHash, + chunks: node.chunks, + inputsDigest: node.inputsDigest, + })), + ...copied, + ]; + if (finished.length) + for (const receipt of await store.finishNodes(finished)) + complete.set(receipt.nodeId, receipt); + if (complete.size) { + if (options.revalidateStoredInputs) + await store.markSelected(journal.snapshot().identity, [...complete.keys()]); + await store.checkpointUnit([...complete.values()]); + } + if (complete.size + excluded.size === unit.nodes.length) { + await journal.acknowledge({ + reusedNodes, + chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0), + }); + } else await journal.fail(failureCause); + progress(); + if (abort) throw abort; + } finally { + for (const row of batch) row.release(); + for (const release of copyReleases) release(); + releaseScratch(); + fetched.release(); + } + }; + + const replay = async (intent: EmbeddingUnitIntent): Promise => { + const units = readEmbeddingWorkUnits(directory, manifest, intent.start); + try { + const first = await units.next(); + if ( + first.done || + first.value.next.offset !== intent.next.offset || + first.value.unit.nodes.length !== intent.nodes + ) + throw new Error('Embedding recovery intent does not match the manifest'); + await processUnit(first.value.unit, intent); + } finally { + await units.return(undefined); + } + }; + + try { + if (journal.snapshot().phase === 'embedding') { + const active = journal.snapshot().active; + if (active) await replay(active); + for await (const intent of journal.failed()) await replay(intent); + let cursor = journal.snapshot().cursor; + for await (const { unit, next } of readEmbeddingWorkUnits(directory, manifest, cursor)) { + await processUnit(unit, { + start: cursor, + next, + nodes: unit.nodes.length, + origin: 'manifest', + }); + cursor = next; + } + if (journal.snapshot().failedUnits) + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes }; + await journal.indexing(); + progress(); + } + signal?.throwIfAborted(); + if (options.revalidateStoredInputs && !store.isSealed) + await store.pruneUnselected( + journal.snapshot().identity, + manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0), + ); + signal?.throwIfAborted(); + const indexKind = await store.seal( + manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0), + journal.snapshot().chunks, + ); + return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind }; + } finally { + sourceReader = undefined; + releaseSourceCache?.(); + } +} diff --git a/gitnexus/src/core/embeddings/embedding-job.ts b/gitnexus/src/core/embeddings/embedding-job.ts new file mode 100644 index 000000000..85a5e34a4 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-job.ts @@ -0,0 +1,626 @@ +/** Common lifecycle for jobs on a published graph. The caller holds its index lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash, randomBytes } from 'node:crypto'; +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { + createEmbeddingJobDirectories, + embeddingJobDirectory, + syncEmbeddingJobDirectory, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../storage/embedding-job.js'; +import { + adoptEmbeddingArtifact, + readEmbeddingArtifactSpace, + type EmbeddingArtifactOwner, +} from '../../storage/embedding-artifact.js'; +import { + EmbeddingJobJournal, + type EmbeddingJobState, +} from '../../storage/embedding-job-journal.js'; +import { + assertOwnedVectorGeneration, + metadataMatchesGraph, +} from '../../storage/vector-generation.js'; +import type { AnalysisResourceBudget } from '../analysis-resource-budget.js'; +import { contentRetentionFromMeta } from '../content-retention.js'; +import { syncPublishedMetadata } from '../analysis-publication.js'; +import { EmbeddingGraphSource } from '../lbug/embedding-source.js'; +import { LegacyVectorSource, legacyVectorCompatibility } from '../lbug/legacy-vector-source.js'; +import { readLegacyEmbeddingSource } from '../../storage/legacy-embedding-source.js'; +import { drainAnalysisNativeResources } from '../lbug/lbug-adapter.js'; +import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; +import { estimateBufferPool } from '../lbug/lbug-config.js'; +import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js'; +import { embeddingJobIdentity } from './embedding-job-identity.js'; +import { resolveEmbeddingIdentity, type EmbeddingIdentity } from './embedding-identity.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { + createEmbeddingWorkManifest, + loadEmbeddingWorkManifest, + type EmbeddingManifestSource, +} from './embedding-work-manifest.js'; +import { runEmbeddingJob } from './embedding-job-runner.js'; +import { publishVectorGeneration, type VectorPublicationActions } from './vector-publication.js'; +import type { EmbeddingConfig, ModelProgressCallback } from './types.js'; +import type { embedBatch } from './embedder.js'; +import type { EmbeddingSidecarReservation } from './embedding-sidecar-client.js'; + +type GraphSource = EmbeddingManifestSource & { close(): Promise }; +export interface PublishedEmbeddingJobOptions { + metadataDirectory: string; + /** Already resolved/owned graph, pinned for the entire job by the index lock. */ + graphPath: string; + resources: AnalysisResourceBudget['embedding']; + config?: Partial; + signal?: AbortSignal; + onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void; + onModelProgress?: ModelProgressCallback; + log?: (message: string) => void; + metadata: VectorPublicationActions; +} + +/** Provider/source seams allow deterministic native-storage restart tests. */ +export interface PublishedEmbeddingJobDependencies { + identity?: () => EmbeddingIdentity; + openSource?: (file: string, poolBytes: number, threads: number) => Promise; + provider?: typeof embedBatch; + initializeProvider?: (config: EmbeddingConfig) => Promise; +} + +export interface PublishedEmbeddingJobResult { + metadata: RepoMeta; + alreadyComplete: boolean; + phase: 'embedding' | 'indexing' | 'ready'; + state?: EmbeddingJobState; + peakQueueBytes: number; +} + +const digest = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); + +async function existsOwnedFile(file: string): Promise { + try { + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Embedding artifact must be an owned regular file'); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } +} + +async function jobDirectory(root: string, identity: string): Promise { + const directory = await embeddingJobDirectory(root); + return createEmbeddingJobDirectories(directory, ['embeddings', identity]); +} + +/** A preflight interrupted before any native/job artifact owns no paid work. + * Only its known draft selection files may be removed. Corrupt paid receipts + * remain evidence and must never turn into a fresh, apparently empty job. + */ +async function clearUnstartedManifest(directory: string): Promise { + if ( + (await existsOwnedFile(path.join(directory, 'vectors'))) || + (await existsOwnedFile(path.join(directory, 'job.json'))) + ) + throw new Error('Embedding manifest is missing for existing paid work; artifacts retained'); + const entries = await fs.opendir(directory); + for await (const entry of entries) { + if (/^(manifest\.pending|manifest\.jsonl|keys-\d+\.jsonl)$/.test(entry.name)) { + const file = path.join(directory, entry.name); + await existsOwnedFile(file); + await fs.unlink(file); + } + } + await syncEmbeddingJobDirectory(directory); +} + +export async function runPublishedEmbeddingJob( + options: PublishedEmbeddingJobOptions, + dependencies: PublishedEmbeddingJobDependencies = {}, +): Promise { + const { resources, signal } = options; + signal?.throwIfAborted(); + // A previous failed close must finish before a model or another native pool + // can consume the next invocation's reservation. + await drainAnalysisNativeResources(); + signal?.throwIfAborted(); + const root = await embeddingJobDirectory(options.metadataDirectory); + // Canonicalize the parent just as we do for the job root. macOS temporary + // paths can name the same directory through /var and /private/var. + const graphPath = path.join( + await embeddingJobDirectory(path.dirname(path.resolve(options.graphPath))), + path.basename(options.graphPath), + ); + let metadata = await options.metadata.loadMetadata(); + const graph = metadata?.graphGeneration; + const runner = metadata?.runnerIdentity; + if ( + !metadata || + !graph || + graph.version !== 1 || + !/^[a-f0-9]{32}$/.test(graph.id) || + !/^[a-f0-9]{64}$/.test(graph.coveredContent) || + !runner || + runner.schemaVersion !== 4 || + !metadata.parseCacheIdentity || + !metadata.schemaFingerprint || + metadata.incrementalInProgress || + metadata.graphWriteCollapsed || + !metadataMatchesGraph(root, metadata, graphPath) + ) + throw new Error('Embedding job requires a verified published graph generation'); + const config = resolveEmbeddingConfig(options.config); + const resolvedVector = (dependencies.identity ?? resolveEmbeddingIdentity)(); + const vector = + dependencies.identity || resolvedVector.provider !== 'local' + ? resolvedVector + : { ...resolvedVector, model: config.modelId }; + if (config.threads > resources.nativeThreads) + throw new Error('Embedding threads exceed the admitted resource budget'); + for (const value of [resources.graphPoolBytes, resources.vectorPoolBytes, resources.queueBytes]) + if (!Number.isSafeInteger(value) || value < 8 * 1024 * 1024) + throw new Error('Invalid embedding resource reservation'); + if (!Number.isSafeInteger(resources.nativeThreads) || resources.nativeThreads < 1) + throw new Error('Invalid embedding CPU reservation'); + const usesLocalRuntime = vector.provider === 'local' && !dependencies.provider; + if ( + usesLocalRuntime && + (!Number.isSafeInteger(resources.modelBytes) || + !Number.isSafeInteger(resources.modelHeapMb) || + resources.modelHeapMb < 128 || + resources.modelHeapMb * 1024 * 1024 + 48 * 1024 * 1024 >= resources.modelBytes) + ) + throw new Error('Local embedding runtime does not fit its resource reservation'); + const identities = embeddingJobIdentity( + { + generation: graph.id, + coveredContent: graph.coveredContent, + revision: metadata.lastCommit || 'non-git', + // Entrypoint paths are diagnostic. Identical artifacts used by analyze, + // sync and the server must resolve to the same paid job identity. + analyzer: digest([ + runner.schemaVersion, + runner.runtime.version, + runner.runtime.platform, + runner.runtime.architecture, + runner.runtime.modulesAbi, + runner.runtime.libc, + runner.cliVersion, + runner.build.digest, + runner.dependencyRuntime.digest, + ]), + parser: digest(metadata.parseCacheIdentity), + schema: metadata.schemaFingerprint, + vectorSchema: VECTOR_STORE_SCHEMA, + }, + vector, + EMBEDDING_TEXT_VERSION, + config, + contentRetentionFromMeta(metadata), + ); + const previous = metadata.vectorGeneration; + if (previous?.identity === identities.identity && !metadata.embeddingJob) { + await assertOwnedVectorGeneration(root, previous.artifactIdentity ?? previous.identity); + return { metadata, alreadyComplete: true, phase: 'ready', peakQueueBytes: 0 }; + } + if (metadata.embeddingJob && metadata.embeddingJob.identity !== identities.identity) { + options.log?.( + metadata.embeddingJob.graphGeneration !== graph.id + ? 'Embedding graph generation changed; prior paid artifacts retained for compatible reuse.' + : 'Embedding configuration changed; prior paid artifacts retained.', + ); + } else if (previous && previous.vectorIdentity !== identities.vectorIdentity) { + options.log?.( + 'Embedding provider, model or dimensions changed; starting a separate vector space.', + ); + } else if (previous && previous.inputIdentity !== identities.inputIdentity) { + options.log?.('Embedding text options changed; prior vectors are not compatible.'); + } + const previousCompatible = + previous && + previous.identity !== identities.identity && + previous.vectorIdentity === identities.vectorIdentity && + previous.inputIdentity === identities.inputIdentity && + previous.dimensions === vector.dimensions; + const limits = (hasFallback: boolean) => { + // Writer and the sole reuse reader share this reservation. A sealed probe + // is physically closed before its replacement writer is opened. + const result = { + bufferPoolBytes: Math.floor(resources.vectorPoolBytes / (hasFallback ? 2 : 1)), + threads: resources.nativeThreads, + batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024), + }; + if (result.bufferPoolBytes < 8 * 1024 * 1024 || result.batchBytes < vector.dimensions * 16) + throw new Error('Vector writer and reuse reader do not fit their resource reservation'); + return result; + }; + const space = { + vectorIdentity: identities.vectorIdentity, + inputIdentity: identities.inputIdentity, + dimensions: vector.dimensions, + schema: VECTOR_STORE_SCHEMA, + }; + const validateOwner = async (owner: EmbeddingArtifactOwner) => { + const directory = await embeddingJobDirectory(path.join(root, 'embeddings', owner.jobIdentity)); + const manifest = await loadEmbeddingWorkManifest(directory, owner.sourceIdentity); + if (manifest.digest !== owner.manifestDigest) + throw new Error('Owned embedding artifact manifest changed; paid artifacts retained'); + }; + let artifact = await readEmbeddingArtifactSpace(root, space, validateOwner); + // A direct legacy receipt survives graph-bound journal replacement. Prefer + // the newer complete artifact when one exists; never open two reuse readers. + const legacyIdentity = + artifact?.owner.fallbackLegacy ?? + (!artifact && !previousCompatible ? metadata.legacyVectorSource : undefined); + let legacy = legacyIdentity ? await readLegacyEmbeddingSource(root, legacyIdentity) : undefined; + if (legacy) { + const incompatible = + legacyVectorCompatibility(legacy.recordedIdentity, vector) ?? + (legacy.recordedInputIdentity !== undefined && + legacy.recordedInputIdentity !== identities.inputIdentity + ? 'Legacy embedding input provenance differs from this request; source retained.' + : undefined); + if (incompatible) { + if (artifact?.owner.fallbackLegacy) throw new Error(incompatible); + options.log?.(incompatible); + legacy = undefined; + } else { + const current = await fs.lstat(graphPath, { bigint: true }); + if (String(current.dev) === legacy.file.dev && String(current.ino) === legacy.file.ino) + throw new Error('Legacy reuse requires a retired source distinct from the mutable graph'); + } + } + if ( + !artifact && + metadata.embeddingJob && + !metadata.embeddingJob.spaceKey && + metadata.embeddingJob.identity !== identities.identity + ) + options.log?.( + 'Prior pending job has no verified vector/input ownership receipt; retaining its files without assuming compatibility.', + ); + const needsFallback = () => + artifact + ? artifact.owner.phase === 'sealed' + ? artifact.owner.jobIdentity !== identities.identity + : Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy) + : Boolean( + legacy || + (previousCompatible && + (previous.artifactIdentity ?? previous.identity) !== identities.identity), + ); + limits(needsFallback()); + const directory = await jobDirectory(root, identities.identity); + let source: GraphSource | undefined; + let store: VectorStore | undefined; + let reuse: VectorStore | LegacyVectorSource | undefined; + let runtime: EmbeddingSidecarReservation | undefined; + try { + if (usesLocalRuntime) { + const { reserveEmbeddingSidecar } = await import('./embedding-sidecar-client.js'); + runtime = await reserveEmbeddingSidecar(config, resources.modelHeapMb); + } else if (!dependencies.provider) { + const { drainEmbeddingSidecarForAnalysis } = await import('./embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + } + // A crash after native sealing but before recording that fact must not + // turn the complete artifact into a mutable writer for a different graph. + if (artifact?.owner.phase === 'pending' && artifact.owner.jobIdentity !== identities.identity) { + const ownedDirectory = await jobDirectory(root, artifact.owner.artifactIdentity); + if (await existsOwnedFile(path.join(ownedDirectory, 'vectors'))) { + store = await VectorStore.open({ + directory: ownedDirectory, + identity: artifact.owner.artifactIdentity, + dimensions: vector.dimensions, + mode: 'resume', + recoverEmptyBootstrap: !(await existsOwnedFile( + path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json'), + )), + ...limits(Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy)), + }); + if (store.isSealed) { + await store.close(); + store = undefined; + artifact = await adoptEmbeddingArtifact( + root, + space, + artifact, + { + ...artifact.owner, + phase: 'sealed', + fallbackArtifact: undefined, + fallbackLegacy: undefined, + }, + validateOwner, + ); + } + } else if ( + await existsOwnedFile(path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json')) + ) { + throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); + } + } + // Sealing may have been discovered by the probe. Admit both pools before + // reading source or creating a replacement manifest. + limits(needsFallback()); + const manifestExists = await existsOwnedFile(path.join(directory, 'manifest.json')); + if (!manifestExists && artifact?.owner.jobIdentity === identities.identity) + throw new Error('Owned embedding manifest is missing; paid artifacts retained'); + if (!manifestExists) await clearUnstartedManifest(directory); + const graphElements = (metadata.stats?.nodes ?? NaN) + (metadata.stats?.edges ?? NaN); + const sourcePoolBytes = + Number.isSafeInteger(graphElements) && graphElements >= 0 + ? Math.min(resources.graphPoolBytes, estimateBufferPool(graphElements)) + : resources.graphPoolBytes; + const getSource = async (): Promise => + (source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)( + graphPath, + sourcePoolBytes, + resources.nativeThreads, + )); + const manifest = manifestExists + ? await loadEmbeddingWorkManifest(directory, identities.sourceIdentity) + : await createEmbeddingWorkManifest(directory, identities.sourceIdentity, await getSource()); + const makeOwner = ( + artifactIdentity: string, + revalidateInputs: boolean, + fallbackArtifact?: string, + fallbackLegacy?: string, + ): EmbeddingArtifactOwner => ({ + artifactIdentity, + jobIdentity: identities.identity, + graphGeneration: graph.id, + sourceIdentity: identities.sourceIdentity, + manifestDigest: manifest.digest, + phase: 'pending', + revalidateInputs, + ...(fallbackArtifact === undefined ? {} : { fallbackArtifact }), + ...(fallbackLegacy === undefined ? {} : { fallbackLegacy }), + }); + let owner: EmbeddingArtifactOwner; + if (artifact?.owner.phase === 'sealed' && artifact.owner.jobIdentity === identities.identity) { + // Publication/index-only recovery for this exact manifest. Never reopen + // a sealed artifact with a native writer handle. + owner = artifact.owner; + } else if (artifact?.owner.phase === 'sealed') { + owner = makeOwner(randomBytes(32).toString('hex'), false, artifact.owner.artifactIdentity); + } else if (artifact) { + owner = makeOwner( + artifact.owner.artifactIdentity, + artifact.owner.revalidateInputs || artifact.owner.jobIdentity !== identities.identity, + artifact.owner.fallbackArtifact, + artifact.owner.fallbackLegacy, + ); + } else { + // Start a space beside its first job. This also adopts an older bounded + // artifact only for this exact compatible job; a digest cannot tell us + // the vector/text provenance of a different old pending job. + const legacyStore = await existsOwnedFile(path.join(directory, 'vectors')); + owner = makeOwner( + identities.identity, + false, + previousCompatible ? (previous.artifactIdentity ?? previous.identity) : undefined, + legacy?.identity, + ); + if (owner.fallbackArtifact === owner.artifactIdentity) owner.fallbackArtifact = undefined; + if ( + legacyStore && + previous?.identity === identities.identity && + (previous.artifactIdentity ?? previous.identity) === owner.artifactIdentity + ) + owner.phase = 'sealed'; + } + artifact = await adoptEmbeddingArtifact(root, space, artifact, owner, validateOwner); + const storeResources = limits(Boolean(owner.fallbackArtifact || owner.fallbackLegacy)); + if ( + metadata.embeddingJob?.identity !== identities.identity || + metadata.embeddingJob.artifactIdentity !== owner.artifactIdentity || + metadata.embeddingJob.spaceKey !== artifact.spaceKey + ) { + metadata = { + ...metadata, + embeddingJob: { + version: 1, + identity: identities.identity, + graphGeneration: graph.id, + artifactIdentity: owner.artifactIdentity, + spaceKey: artifact.spaceKey, + }, + }; + await options.metadata.saveMetadata(metadata); + await syncPublishedMetadata(root, metadata); + } + signal?.throwIfAborted(); + const artifactDirectory = await jobDirectory(root, owner.artifactIdentity); + if (owner.artifactIdentity !== owner.jobIdentity) { + // Rebased artifacts live apart from the journal. Keep independent proof + // of ownership so a later generation can reclaim their native payload. + try { + const receipt = (await readEmbeddingJobRecord(artifactDirectory, 'artifact.json')) as { + version: number; + identity: string; + jobIdentity: string; + }; + if ( + receipt.version !== 1 || + receipt.identity !== owner.artifactIdentity || + !/^[a-f0-9]{64}$/.test(receipt.jobIdentity) + ) + throw new Error('Embedding artifact has unknown ownership'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + await writeEmbeddingJobRecord(artifactDirectory, 'artifact.json', { + version: 1, + identity: owner.artifactIdentity, + jobIdentity: owner.jobIdentity, + }); + } + } + const hasStore = await existsOwnedFile(path.join(artifactDirectory, 'vectors')); + const hasJournal = await existsOwnedFile(path.join(directory, 'job.json')); + if (!hasStore && hasJournal) + throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained'); + store ??= await VectorStore.open({ + directory: artifactDirectory, + identity: owner.artifactIdentity, + dimensions: vector.dimensions, + mode: owner.phase === 'sealed' ? 'read' : hasStore ? 'resume' : 'create', + recoverEmptyBootstrap: !hasJournal, + ...storeResources, + }); + const journal = await EmbeddingJobJournal.recover(directory, identities.identity, manifest); + if (owner.fallbackArtifact) { + const file = await assertOwnedVectorGeneration(root, owner.fallbackArtifact); + reuse = await VectorStore.open({ + directory: path.dirname(file), + identity: owner.fallbackArtifact, + dimensions: vector.dimensions, + mode: 'read', + ...storeResources, + }); + } else if (owner.fallbackLegacy && !store.isSealed) { + if (!legacy || legacy.identity !== owner.fallbackLegacy) + throw new Error('Legacy source does not match the owned fallback; artifacts retained'); + reuse = await LegacyVectorSource.open( + { + path: legacy.path, + sourceIdentity: legacy.identity, + recordedIdentity: legacy.recordedIdentity, + expectedIdentity: vector, + recordedInputIdentity: legacy.recordedInputIdentity, + expectedInputIdentity: identities.inputIdentity, + config, + signal, + log: options.log, + ...storeResources, + }, + store, + legacy.excludedIds, + ); + } + const provider: typeof embedBatch = + dependencies.provider ?? + (async (texts, request) => { + if (runtime) return runtime.embedBatch(texts, request); + const { embedBatch } = await import('./embedder.js'); + return embedBatch(texts, { + ...request, + maxResponseBytes: texts.length * (4096 + 32 * vector.dimensions), + }); + }); + const initializeProvider = async () => { + if (dependencies.initializeProvider) return dependencies.initializeProvider(config); + await runtime?.initialize(options.onModelProgress); + }; + const result = await runEmbeddingJob({ + directory, + manifest, + journal, + store, + reuse, + // A sealed artifact still belongs to this exact job/manifest; rebuilding + // a damaged cursor can use its complete groups without source reads. + revalidateStoredInputs: owner.revalidateInputs && !store.isSealed, + config, + queueBytes: resources.queueBytes, + sourceContext: { + repoPath: metadata.repoPath, + fileHashes: metadata.fileHashes, + contentRetention: contentRetentionFromMeta(metadata), + }, + // Opening the graph is lazy even on resume: indexing-only retries do not + // enumerate/fetch source or initialize a provider. + source: { + scan: async function* (label) { + yield* (await getSource()).scan(label); + }, + fetch: async (label, ids) => (await getSource()).fetch(label, ids), + }, + provider, + initializeProvider, + signal, + onProgress: options.onProgress, + }); + const sealed = store.isSealed; + await store.close(); + store = undefined; + await reuse?.close(); + reuse = undefined; + await source?.close(); + source = undefined; + signal?.throwIfAborted(); + if (!result.indexKind) + return { metadata, alreadyComplete: false, phase: 'embedding', ...result }; + if (sealed && artifact.owner.phase !== 'sealed') { + artifact = await adoptEmbeddingArtifact( + root, + space, + artifact, + { + ...artifact.owner, + phase: 'sealed', + fallbackArtifact: undefined, + fallbackLegacy: undefined, + }, + validateOwner, + ); + } + const published = await publishVectorGeneration( + root, + { + version: 1, + identity: identities.identity, + artifactIdentity: owner.artifactIdentity, + graphGeneration: graph.id, + vectorIdentity: identities.vectorIdentity, + inputIdentity: identities.inputIdentity, + dimensions: vector.dimensions, + nodes: manifest.totalNodes, + chunks: result.state.chunks, + ...(result.state.excludedNodes + ? { + excludedNodes: result.state.excludedNodes, + exclusionReason: 'input-rejected' as const, + } + : {}), + indexKind: result.indexKind, + }, + storeResources, + options.metadata, + legacy?.identity, + ); + if (!published) return { metadata, alreadyComplete: false, phase: 'indexing', ...result }; + if (result.state.excludedNodes) + options.log?.( + `Skipped ${result.state.excludedNodes} node(s) after three deterministic embedding input rejections; compatible vectors were published.`, + ); + if (journal.snapshot().phase !== 'ready') await journal.ready(); + options.onProgress?.(journal.snapshot(), result.peakQueueBytes); + return { + metadata: published, + alreadyComplete: false, + phase: 'ready', + state: journal.snapshot(), + peakQueueBytes: result.peakQueueBytes, + }; + } finally { + // Sequential finally blocks close every native handle even when one close fails. + try { + await store?.close(); + } finally { + try { + await reuse?.close(); + } finally { + try { + await source?.close(); + } finally { + runtime?.release(); + } + } + } + } +} diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 9b70bddaa..a254c20c3 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -65,6 +65,7 @@ import { needsEmbeddingSource, type EmbeddingSourceContext, } from './source-reader.js'; +import type { VectorSearchSnapshot } from './vector-generation-search.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -115,7 +116,6 @@ export const contentHashForNode = ( ): string => { // Hash must be deterministic across runs, so exclude methodNames/fieldNames // which are populated during the batch loop via AST extraction. - // Using only node.content ensures the hash stays stable. // NOTE: A change to extractStructuralNames behavior requires bumping EMBEDDING_TEXT_VERSION. const hashNode = { ...node, methodNames: undefined, fieldNames: undefined }; if (node.label === 'Class') hashNode.classDeclaration = undefined; @@ -181,7 +181,7 @@ export const contentHashForNode = ( return hash.digest('hex'); }; -const isInputRejection = (error: unknown): boolean => +export const isInputRejection = (error: unknown): boolean => error instanceof HttpEmbeddingError && (error.status === 400 || error.status === 413 || error.status === 422); @@ -193,14 +193,15 @@ type EmbeddedRange = { start: number; end: number } & ( /** Bisect only input rejections, with at most 2n-1 requests per sub-batch. * Authentication, throttling, transport and cancellation retain their existing * bounded retry/failure policy instead of multiplying requests. */ -async function* embedInputRanges( +export async function* embedInputRanges( texts: string[], subBatchSize: number, signal?: AbortSignal, + provider: typeof embedBatch = embedBatch, ): AsyncGenerator { async function* request(start: number, end: number): AsyncGenerator { try { - const embeddings = await embedBatch(texts.slice(start, end), { signal }); + const embeddings = await provider(texts.slice(start, end), { signal }); yield { start, end, embeddings }; } catch (error) { if (signal?.aborted || isCancellationError(error)) throw error; @@ -1280,34 +1281,45 @@ export const semanticSearch = async ( query: string, k: number = 10, maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), + snapshot?: VectorSearchSnapshot, + embedQuery?: (text: string) => Promise, ): Promise => { - // determinism: probe — existence only. Only `exists.length` is read; which - // row LIMIT 1 returns cannot change whether the table is empty. - const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`); - if (!exists.length) { - return []; - } - - if (!isEmbedderReady()) { - throw new Error('Embedding model not initialized. Run embedding pipeline first.'); - } - - const queryEmbedding = await embedText(query); - const queryVec = embeddingToArray(queryEmbedding); - const queryVecStr = `[${queryVec.join(',')}]`; - + const queryVector = async (): Promise => { + if (embedQuery) return [...(await embedQuery(query))]; + if (!isEmbedderReady()) + throw new Error('Embedding model not initialized. Run embedding pipeline first.'); + return embeddingToArray(await embedText(query)); + }; let bestChunks = new Map< string, { distance: number; chunkIndex: number; startLine: number; endLine: number } >(); - // Query/read path: NEVER spawn a network INSTALL on a user query. If the - // VECTOR extension was not pre-installed, fall back to exact scan rather than - // blocking the query on a download (offline-first; see extension-loader.ts - // "load-only" — used by all serve/MCP query paths). - if (await loadVectorExtension(undefined, { policy: 'load-only' })) { - try { - bestChunks = await collectBestChunks(k, async (fetchLimit) => { - const vectorQuery = ` + let generationSelected = false; + if (snapshot) { + const { searchVectorGeneration } = await import('./vector-generation-search.js'); + const chunks = await searchVectorGeneration(snapshot, queryVector, k, maxDistance); + generationSelected = chunks !== null; + if (chunks) bestChunks = new Map(chunks.map(({ nodeId, ...chunk }) => [nodeId, chunk])); + } + if (!generationSelected) { + // determinism: probe — existence only. Only `exists.length` is read; which + // row LIMIT 1 returns cannot change whether the table is empty. + const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`); + if (!exists.length) { + return []; + } + + const queryVec = await queryVector(); + const queryVecStr = `[${queryVec.join(',')}]`; + + // Query/read path: NEVER spawn a network INSTALL on a user query. If the + // VECTOR extension was not pre-installed, fall back to exact scan rather than + // blocking the query on a download (offline-first; see extension-loader.ts + // "load-only" — used by all serve/MCP query paths). + if (await loadVectorExtension(undefined, { policy: 'load-only' })) { + try { + bestChunks = await collectBestChunks(k, async (fetchLimit) => { + const vectorQuery = ` CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit}) YIELD node AS emb, distance @@ -1318,54 +1330,54 @@ export const semanticSearch = async ( ORDER BY distance `; - const embResults = await executeQuery(vectorQuery); - return embResults.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - distance: row.distance ?? row[4], - })); - }); - } catch { - bestChunks = new Map(); + const embResults = await executeQuery(vectorQuery); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); + } catch { + bestChunks = new Map(); + } } - } - if (bestChunks.size === 0) { - // NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher - // never writes a fabricated 0, whereas here `?? 0` is the right answer — - // an unknown count simply skips the exact scan. - const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); - const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0); - const exactLimit = getExactScanLimit(); - if (embeddingCount > 0 && embeddingCount <= exactLimit) { - const rows = await executeQuery(` + if (bestChunks.size === 0) { + // NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher + // never writes a fabricated 0, whereas here `?? 0` is the right answer — + // an unknown count simply skips the exact scan. + const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); + const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0); + const exactLimit = getExactScanLimit(); + if (embeddingCount > 0 && embeddingCount <= exactLimit) { + const rows = await executeQuery(` MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding `); - const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - embedding: row.embedding ?? row[4] ?? [], - })); - bestChunks = new Map( - rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [ - row.nodeId, - { - distance: row.distance, - chunkIndex: row.chunkIndex, - startLine: row.startLine, - endLine: row.endLine, - }, - ]), - ); + const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + embedding: row.embedding ?? row[4] ?? [], + })); + bestChunks = new Map( + rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [ + row.nodeId, + { + distance: row.distance, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + }, + ]), + ); + } } } - if (bestChunks.size === 0) { return []; } @@ -1431,8 +1443,9 @@ export const semanticSearchWithContext = async ( query: string, k: number = 5, _hops: number = 1, + snapshot?: VectorSearchSnapshot, ): Promise => { - const results = await semanticSearch(executeQuery, query, k); + const results = await semanticSearch(executeQuery, query, k, undefined, snapshot); return results.map((r) => ({ matchId: r.nodeId, diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts index ce8e1e2f0..a78e307db 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts @@ -24,6 +24,8 @@ import type { SidecarResponse, } from './embedding-sidecar-protocol.js'; import { logger } from '../logger.js'; +import { isDeepStrictEqual } from 'node:util'; +import { resolveEmbeddingConfig } from './config.js'; export type ForkImpl = ( modulePath: string, @@ -47,10 +49,16 @@ let lastInitOptions: | { embeddingConfig?: Partial; forceDevice?: EmbeddingSidecarDevice; + heapLimitMb?: number; } | undefined; let exitHooked = false; let progressSink: ((progress: ModelProgress) => void) | undefined; +// A caller abort only stops waiting. The child may still be inside ONNX; keep +// its physical requests and process allocation until a response/close proves release. +const physicalRequests = new Map(); +const liveChildren = new Set(); +let runtimeReservation: { config: EmbeddingConfig; heapLimitMb: number } | undefined; const pending = new Map< number, @@ -75,6 +83,9 @@ export const _resetEmbeddingSidecarForTests = (): void => { device = 'cpu'; ensureChain = null; lastInitOptions = undefined; + runtimeReservation = undefined; + physicalRequests.clear(); + liveChildren.clear(); }; const sidecarScriptPath = (): string => { @@ -91,9 +102,26 @@ const tsxHookArgs = (): string[] => { return ['--import', pathToFileURL(require.resolve('tsx/esm')).href]; }; -const childEnv = (): NodeJS.ProcessEnv => { +const childEnv = (configPinned = false): NodeJS.ProcessEnv => { const env = { ...process.env }; delete env.GITNEXUS_EMBEDDING_URL; + if (configPinned) { + // The admitted, resolved config travels over IPC. An environment mutation + // between admission and lazy model init must not override its allocations. + for (const name of [ + 'THREADS', + 'DEVICE', + 'BATCH_SIZE', + 'SUB_BATCH_SIZE', + 'MAX_INPUT_CHARS', + 'MODEL_ID', + 'CHUNK_SIZE', + 'CHUNK_OVERLAP', + 'CHUNK_STRATEGY', + ]) { + delete env[`GITNEXUS_EMBEDDING_${name}`]; + } + } return env; }; @@ -146,10 +174,14 @@ const rejectAll = (error: Error): void => { }; const attachChild = (proc: ChildProcess): void => { + liveChildren.add(proc); proc.stderr?.on('data', (chunk: Buffer | string) => { logger.debug({ sidecar: true }, String(chunk).trimEnd()); }); proc.on('message', (msg: SidecarResponse) => { + if (msg.type !== 'progress' && physicalRequests.get(msg.id) === proc) + physicalRequests.delete(msg.id); + if (child !== proc) return; if (msg.type === 'progress') { progressSink?.({ status: msg.status, @@ -162,6 +194,8 @@ const attachChild = (proc: ChildProcess): void => { waiter.resolve(msg); }); proc.on('close', (code, signal) => { + liveChildren.delete(proc); + for (const [id, owner] of physicalRequests) if (owner === proc) physicalRequests.delete(id); if (child !== proc) return; noteChildDeath(signal); child = null; @@ -183,6 +217,7 @@ const request = ( signal?: AbortSignal, ): Promise => { if (!child) return Promise.reject(new Error('Embedding sidecar is not running')); + const proc = child; try { signal?.throwIfAborted(); } catch (err) { @@ -210,7 +245,7 @@ const request = ( }; const timer = setTimeout(() => { finish(() => { - child?.kill('SIGKILL'); + proc.kill('SIGKILL'); reject(new Error(`Embedding sidecar request timed out after ${timeoutMs}ms (${msg.type})`)); }); }, timeoutMs); @@ -225,8 +260,10 @@ const request = ( return; } try { - child!.send({ ...msg, id } as SidecarRequest); + physicalRequests.set(id, proc); + proc.send({ ...msg, id } as SidecarRequest); } catch (err) { + physicalRequests.delete(id); finish(() => { reject(err instanceof Error ? err : new Error(String(err))); }); @@ -234,11 +271,19 @@ const request = ( }); }; -const spawnSidecar = (): ChildProcess => { +const spawnSidecar = (heapLimitMb?: number): ChildProcess => { + if (liveChildren.size > 0) { + throw new Error('Previous embedding sidecar has not released its native resource reservation'); + } const proc = forkImpl(sidecarScriptPath(), [], { - execArgv: tsxHookArgs(), + execArgv: [ + ...tsxHookArgs(), + ...(heapLimitMb === undefined + ? [] + : [`--max-old-space-size=${heapLimitMb}`, '--max-semi-space-size=16']), + ], stdio: ['ignore', 'ignore', 'pipe', 'ipc'], - env: childEnv(), + env: childEnv(runtimeReservation !== undefined), }); if (!exitHooked) { exitHooked = true; @@ -263,21 +308,32 @@ export const reapEmbeddingSidecar = (): void => { }; /** Reap and wait for the killed child's close/error so an awaited dispose is a real boundary. */ -export const reapEmbeddingSidecarAndWait = async (timeoutMs = 5_000): Promise => { +export const reapEmbeddingSidecarAndWait = async ( + timeoutMs = 5_000, + options: { requireExit?: boolean } = {}, +): Promise => { const proc = child; if (!proc) return; const closed = new Promise((resolve) => { const finish = (): void => resolve(); proc.once('close', finish); - proc.once('error', finish); + if (!options.requireExit) proc.once('error', finish); }); reapEmbeddingSidecar(); let timer: ReturnType | undefined; try { await Promise.race([ closed, - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs); + new Promise((resolve, reject) => { + timer = setTimeout( + () => + options.requireExit + ? reject( + new Error('Embedding sidecar has not released its native resource reservation'), + ) + : resolve(), + timeoutMs, + ); }), ]); } finally { @@ -314,11 +370,15 @@ const spawnAndInit = async (options?: EnsureSidecarOptions): Promise => { } const persisted = { - embeddingConfig: options?.embeddingConfig ?? lastInitOptions?.embeddingConfig, - forceDevice: options?.forceDevice ?? lastInitOptions?.forceDevice, + embeddingConfig: + runtimeReservation?.config ?? + resolveEmbeddingConfig(options?.embeddingConfig ?? lastInitOptions?.embeddingConfig), + forceDevice: + options?.forceDevice ?? (runtimeReservation ? undefined : lastInitOptions?.forceDevice), + heapLimitMb: runtimeReservation?.heapLimitMb ?? lastInitOptions?.heapLimitMb, }; - child = spawnSidecar(); + child = spawnSidecar(persisted.heapLimitMb); attachChild(child); progressSink = options?.onProgress; try { @@ -386,6 +446,26 @@ const raceAbort = async (promise: Promise, signal?: AbortSignal): Promise< export const ensureEmbeddingSidecar = async ( options?: EnsureSidecarOptions, ): Promise<{ device: EmbeddingSidecarDevice }> => { + if (runtimeReservation) { + if ( + options?.forceDevice && + runtimeReservation.config.device !== 'auto' && + options.forceDevice !== runtimeReservation.config.device + ) { + throw new Error('Embedding device conflicts with an active job resource reservation'); + } + for (const [key, value] of Object.entries(options?.embeddingConfig ?? {})) { + if ( + value !== undefined && + value !== runtimeReservation.config[key as keyof EmbeddingConfig] + ) { + throw new Error( + 'Embedding configuration conflicts with an active job resource reservation', + ); + } + } + options = { ...options, embeddingConfig: runtimeReservation.config }; + } if (localUnavailable) throw localUnavailableError(); if (ready && child) { rejectConflictingForceDevice(options?.forceDevice); @@ -403,6 +483,69 @@ export const ensureEmbeddingSidecar = async ( return { device }; }; +export interface EmbeddingSidecarReservation { + initialize(onProgress?: (progress: ModelProgress) => void): Promise; + embedBatch(texts: string[], options?: { signal?: AbortSignal }): Promise; + release(): void; +} + +/** Admit before opening job databases; acquire does not load a model. A warm + * compatible sidecar remains usable, and only one job can reconfigure it. */ +export async function reserveEmbeddingSidecar( + config: EmbeddingConfig, + heapLimitMb: number, +): Promise { + if (!Number.isSafeInteger(heapLimitMb) || heapLimitMb < 128) + throw new Error('Embedding sidecar heap allocation must be at least 128 MiB'); + if (runtimeReservation || ensureChain || physicalRequests.size > 0) + throw new Error('Embedding runtime is busy; cannot reassign its resource reservation'); + const reservation = { config: Object.freeze({ ...config }), heapLimitMb }; + runtimeReservation = reservation; + try { + if ( + child && + (!ready || + lastInitOptions?.heapLimitMb !== heapLimitMb || + !isDeepStrictEqual(lastInitOptions?.embeddingConfig, config)) + ) { + await reapEmbeddingSidecarAndWait(5_000, { requireExit: true }); + } + if (liveChildren.size > (child ? 1 : 0)) + throw new Error( + 'Previous embedding sidecar has not released its native resource reservation', + ); + const assertOwned = () => { + if (runtimeReservation !== reservation) + throw new Error('Embedding runtime reservation was released'); + }; + return { + initialize: async (onProgress) => { + assertOwned(); + await ensureEmbeddingSidecar({ onProgress, embeddingConfig: reservation.config }); + }, + embedBatch: async (texts, options) => { + assertOwned(); + return sidecarEmbedBatch(texts, options); + }, + release: () => { + if (runtimeReservation === reservation) runtimeReservation = undefined; + }, + }; + } catch (error) { + if (runtimeReservation === reservation) runtimeReservation = undefined; + throw error; + } +} + +/** Parser phases reuse the model reservation only after an idle child exits. */ +export async function drainEmbeddingSidecarForAnalysis(): Promise { + if (runtimeReservation || ensureChain || physicalRequests.size > 0) + throw new Error('Embedding runtime is busy; refusing to reuse its native resource reservation'); + await reapEmbeddingSidecarAndWait(5_000, { requireExit: true }); + if (liveChildren.size > 0) + throw new Error('Previous embedding sidecar has not released its native resource reservation'); +} + const vectorsFromEmbedResponse = (response: SidecarResponse): Float32Array[] => { if (response.type === 'error') throw new Error(response.message); if (response.type !== 'vectors') { diff --git a/gitnexus/src/core/embeddings/embedding-sidecar.ts b/gitnexus/src/core/embeddings/embedding-sidecar.ts index 70f3aa936..93f0de9c8 100644 --- a/gitnexus/src/core/embeddings/embedding-sidecar.ts +++ b/gitnexus/src/core/embeddings/embedding-sidecar.ts @@ -16,6 +16,10 @@ process.on('message', (msg: SidecarRequest) => { void handle(msg); }); +// This is a standalone process, not a worker thread. If its owner exits or is +// killed after cancellation grace, release the model instead of orphaning it. +process.once('disconnect', () => process.exit(1)); + async function handle(msg: SidecarRequest): Promise { try { switch (msg.type) { diff --git a/gitnexus/src/core/embeddings/embedding-work-manifest.ts b/gitnexus/src/core/embeddings/embedding-work-manifest.ts new file mode 100644 index 000000000..9001d0e16 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-work-manifest.ts @@ -0,0 +1,385 @@ +import { createHash } from 'node:crypto'; +import fs, { type FileHandle } from 'node:fs/promises'; +import path from 'node:path'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import { EmbeddingByteBudget } from '../analysis-resource-budget.js'; +import { EMBEDDABLE_LABELS, type EmbeddableNode } from './types.js'; +import { + embeddingJobDirectory, + openEmbeddingJobFile, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from '../../storage/embedding-job.js'; + +// Persisted unit identity must not depend on a retry's memory/batch settings. +const UNIT_NODES = 64; +const UNIT_SOURCE_BYTES = 1024 * 1024; +const MAX_LINE_BYTES = 1024 * 1024; +const IO_BYTES = 64 * 1024; +const LABELS = new Set([...EMBEDDABLE_LABELS, 'File']); + +export interface EmbeddingWorkKey { + id: string; + contentBytes: number; + /** Upper bound for serialized persisted node metadata, including its ID. */ + metadataBytes: number; +} + +export interface EmbeddingManifestSource { + /** Key-only scan in UTF-8 ID order. File scans yield only eligible text files. */ + scan(label: string): AsyncIterable; + /** Bounded primary-key fetch against the same immutable graph generation. */ + fetch(label: string, ids: readonly string[]): Promise; +} + +export interface EmbeddingWorkUnit { + ordinal: number; + label: string; + nodes: EmbeddingWorkKey[]; + /** Source strings plus persisted metadata; excludes later text/vector queues. */ + retainedBytes: number; +} + +export interface EmbeddingManifestReceipt { + version: 1; + sourceIdentity: string; + digest: string; + bytes: number; + totalNodes: number; + totalUnits: number; + maximumUnitBytes: number; +} + +export interface EmbeddingManifestCursor { + unit: number; + offset: number; +} + +function count(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} + +function keyBytes(key: EmbeddingWorkKey): number { + return 2 * (key.contentBytes + key.metadataBytes); +} + +function validateKey(value: unknown): asserts value is EmbeddingWorkKey { + if (!value || typeof value !== 'object' || !('id' in value)) + throw new Error('Malformed embedding manifest key'); + assertValidGraphNodeId(value.id, 'Embedding manifest'); + if ( + Buffer.from(value.id).toString('utf8') !== value.id || + !('contentBytes' in value) || + !count(value.contentBytes) || + !('metadataBytes' in value) || + !count(value.metadataBytes) || + value.metadataBytes < Buffer.byteLength(value.id) + ) { + throw new Error('Invalid embedding manifest key bounds'); + } + if (!Number.isSafeInteger(2 * (value.contentBytes + value.metadataBytes))) + throw new Error('Embedding key size exceeds safe accounting'); +} + +class BufferedLines { + private pending: string[] = []; + private size = 0; + constructor(private readonly handle: FileHandle) {} + async write(line: string): Promise { + const size = Buffer.byteLength(line); + if (size > MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + this.pending.push(line); + this.size += size; + if (this.size >= IO_BYTES) await this.flush(); + } + async flush(): Promise { + if (!this.size) return; + await this.handle.writeFile(this.pending.join('')); + this.pending = []; + this.size = 0; + } +} + +/** Framing stays bounded even for a corrupt file without a newline. */ +async function* lines(file: string, start = 0): AsyncGenerator<{ text: string; end: number }> { + const { handle } = await openEmbeddingJobFile(file); + try { + const buffer = Buffer.alloc(IO_BYTES); + let pending = Buffer.alloc(0); + let position = start; + let lineStart = start; + for (;;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (!bytesRead) break; + position += bytesRead; + const data = pending.length + ? Buffer.concat([pending, buffer.subarray(0, bytesRead)]) + : buffer.subarray(0, bytesRead); + let offset = 0; + for (;;) { + const end = data.indexOf(10, offset); + if (end < 0) break; + if (end - offset + 1 > MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + lineStart += end - offset + 1; + yield { + text: new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)), + end: lineStart, + }; + offset = end + 1; + } + pending = Buffer.from(data.subarray(offset)); + if (pending.length >= MAX_LINE_BYTES) + throw new Error('Embedding manifest record exceeds its bounded size'); + } + if (pending.length) throw new Error('Embedding manifest has an incomplete trailing record'); + } finally { + await handle.close(); + } +} + +/** Merge label-key spools to detect cross-label duplicates without a corpus Set. */ +async function verifyUniqueKeys(files: string[]): Promise { + const readers = files.map((file) => lines(file)); + const heads: Array<{ id: string; bytes: Buffer } | undefined> = []; + const head = (text: string): { id: string; bytes: Buffer } => { + const { id }: { id: string } = JSON.parse(text); + return { id, bytes: Buffer.from(id) }; + }; + try { + for (const reader of readers) { + const next = await reader.next(); + heads.push(next.done ? undefined : head(next.value.text)); + } + let previous: string | undefined; + for (;;) { + let selected = -1; + let selectedHead: { id: string; bytes: Buffer } | undefined; + for (let i = 0; i < heads.length; i++) { + const candidate = heads[i]; + if ( + candidate && + (!selectedHead || Buffer.compare(candidate.bytes, selectedHead.bytes) < 0) + ) { + selected = i; + selectedHead = candidate; + } + } + if (!selectedHead) break; + const id = selectedHead.id; + if (id === previous) + throw new Error( + 'Embedding read: duplicate graph node identity; rebuild with analyze --force', + ); + previous = id; + const next = await readers[selected].next(); + heads[selected] = next.done ? undefined : head(next.value.text); + } + } finally { + await Promise.all(readers.map((reader) => reader.return(undefined))); + } +} + +/** Preflight and publish the complete selection before any provider/write work. */ +export async function createEmbeddingWorkManifest( + directory: string, + sourceIdentity: string, + source: EmbeddingManifestSource, +): Promise { + if (!sourceIdentity || Buffer.byteLength(sourceIdentity) > 512) + throw new Error('Invalid manifest source identity'); + const canonical = await embeddingJobDirectory(directory); + const pendingPath = path.join(canonical, 'manifest.pending'); + const handle = await fs.open(pendingPath, 'wx', 0o600); + const output = new BufferedLines(handle); + const hash = createHash('sha256'); + const keyFiles: string[] = []; + let totalNodes = 0; + let totalUnits = 0; + let maximumUnitBytes = 0; + let bytes = 0; + try { + const scanLabel = async (label: string): Promise => { + const keyPath = path.join(canonical, `keys-${keyFiles.length}.jsonl`); + const keysHandle = await fs.open(keyPath, 'wx', 0o600); + keyFiles.push(keyPath); + const keyOutput = new BufferedLines(keysHandle); + let previous: Buffer | undefined; + let nodes: EmbeddingWorkKey[] = []; + let retainedBytes = 0; + let recordBytes = 0; + const flush = async (): Promise => { + if (!nodes.length) return; + const unit: EmbeddingWorkUnit = { ordinal: totalUnits++, label, nodes, retainedBytes }; + const line = `${JSON.stringify(unit)}\n`; + await output.write(line); + hash.update(line); + bytes += Buffer.byteLength(line); + maximumUnitBytes = Math.max(maximumUnitBytes, retainedBytes); + nodes = []; + retainedBytes = 0; + recordBytes = 0; + }; + try { + for await (const key of source.scan(label)) { + validateKey(key); + const id = Buffer.from(key.id); + if (previous && Buffer.compare(previous, id) >= 0) + throw new Error( + `Embedding ${label} key scan is unordered or contains duplicate identities`, + ); + previous = id; + const serialized = `${JSON.stringify(key)}\n`; + const size = Buffer.byteLength(serialized); + if (size + 1024 > MAX_LINE_BYTES) + throw new Error('Embedding identity exceeds the manifest record budget'); + if ( + nodes.length && + (nodes.length === UNIT_NODES || + retainedBytes + keyBytes(key) > UNIT_SOURCE_BYTES || + recordBytes + size + 1024 > MAX_LINE_BYTES) + ) + await flush(); + await keyOutput.write(serialized); + nodes.push({ + id: key.id, + contentBytes: key.contentBytes, + metadataBytes: key.metadataBytes, + }); + retainedBytes += keyBytes(key); + recordBytes += size; + totalNodes++; + } + await flush(); + await keyOutput.flush(); + } finally { + await keysHandle.close(); + } + }; + for (const label of EMBEDDABLE_LABELS) await scanLabel(label); + if (totalNodes === 0) await scanLabel('File'); + await verifyUniqueKeys(keyFiles); + await output.flush(); + await handle.sync(); + } finally { + await handle.close(); + } + const receipt: EmbeddingManifestReceipt = { + version: 1, + sourceIdentity, + digest: hash.digest('hex'), + bytes, + totalNodes, + totalUnits, + maximumUnitBytes, + }; + await fs.rename(pendingPath, path.join(canonical, 'manifest.jsonl')); + await syncEmbeddingJobDirectory(canonical); + await writeEmbeddingJobRecord(canonical, 'manifest.json', receipt); + await Promise.all(keyFiles.map((file) => fs.unlink(file))); + return receipt; +} + +/** Validate the durable selection without re-reading graph content or keys. */ +export async function loadEmbeddingWorkManifest( + directory: string, + sourceIdentity: string, +): Promise { + const value = await readEmbeddingJobRecord(directory, 'manifest.json'); + if (!value || typeof value !== 'object') throw new Error('Malformed embedding manifest receipt'); + const receipt = value as EmbeddingManifestReceipt; + if ( + receipt.version !== 1 || + receipt.sourceIdentity !== sourceIdentity || + !/^[0-9a-f]{64}$/.test(receipt.digest) || + !count(receipt.bytes) || + !count(receipt.totalNodes) || + !count(receipt.totalUnits) || + !count(receipt.maximumUnitBytes) + ) { + throw new Error('Embedding manifest identity or version mismatch; artifacts retained'); + } + const canonical = await embeddingJobDirectory(directory); + const hash = createHash('sha256'); + let bytes = 0; + for await (const line of lines(path.join(canonical, 'manifest.jsonl'))) { + hash.update(`${line.text}\n`); + bytes = line.end; + if (bytes > receipt.bytes) throw new Error('Embedding manifest size mismatch'); + } + if (bytes !== receipt.bytes || hash.digest('hex') !== receipt.digest) + throw new Error('Embedding manifest checksum mismatch; artifacts retained'); + return receipt; +} + +export async function* readEmbeddingWorkUnits( + directory: string, + receipt: EmbeddingManifestReceipt, + cursor: EmbeddingManifestCursor = { unit: 0, offset: 0 }, +): AsyncGenerator<{ unit: EmbeddingWorkUnit; next: EmbeddingManifestCursor }> { + if ( + !count(cursor.unit) || + cursor.unit > receipt.totalUnits || + !count(cursor.offset) || + cursor.offset > receipt.bytes + ) + throw new Error('Invalid embedding manifest cursor'); + const canonical = await embeddingJobDirectory(directory); + let ordinal = cursor.unit; + for await (const line of lines(path.join(canonical, 'manifest.jsonl'), cursor.offset)) { + const unit: EmbeddingWorkUnit = JSON.parse(line.text); + if ( + unit.ordinal !== ordinal || + !LABELS.has(unit.label) || + !Array.isArray(unit.nodes) || + !unit.nodes.length || + unit.nodes.length > UNIT_NODES + ) + throw new Error('Invalid embedding work unit'); + unit.nodes.forEach(validateKey); + if ( + unit.retainedBytes !== unit.nodes.reduce((sum, key) => sum + keyBytes(key), 0) || + unit.retainedBytes > receipt.maximumUnitBytes + ) + throw new Error('Invalid embedding unit byte accounting'); + ordinal++; + yield { unit, next: { unit: ordinal, offset: line.end } }; + } + if (ordinal !== receipt.totalUnits) + throw new Error('Embedding manifest cursor does not cover the remaining units'); +} + +export async function fetchEmbeddingWorkUnit( + source: EmbeddingManifestSource, + unit: EmbeddingWorkUnit, + budget: EmbeddingByteBudget, +): Promise<{ nodes: EmbeddableNode[]; release: () => void }> { + const release = budget.reserve(unit.retainedBytes); + try { + const rows = await source.fetch( + unit.label, + unit.nodes.map((node) => node.id), + ); + const byId = new Map(rows.map((row) => [row.id, row])); + if (rows.length !== unit.nodes.length || byId.size !== rows.length) + throw new Error('Embedding source returned missing or duplicate nodes'); + const nodes = unit.nodes.map((key) => { + const node = byId.get(key.id); + if ( + !node || + node.label !== unit.label || + typeof node.content !== 'string' || + Buffer.byteLength(node.content) !== key.contentBytes || + Buffer.byteLength(JSON.stringify({ ...node, content: undefined })) > key.metadataBytes + ) + throw new Error('Embedding graph changed after manifest publication'); + return node; + }); + return { nodes, release }; + } catch (error) { + release(); + throw error; + } +} diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 689d3cf1d..7ed012028 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -46,6 +46,46 @@ interface HttpConfig { export interface EmbeddingRequestOptions { signal?: AbortSignal; + /** Encoded successful response bound, reserved by the calling job. */ + maxResponseBytes?: number; +} + +// A local resource limit aborts the request without retrying or penalising +// endpoint health. resilientFetch treats AbortError DOMExceptions as neutral. +class EmbeddingResponseBudgetError extends DOMException { + constructor(message: string) { + super(message, 'AbortError'); + } +} + +/** Bound encoded bytes before JSON decoding; Content-Length alone is untrusted. */ +async function readEmbeddingResponse(response: Response, maximum?: number): Promise { + if (maximum === undefined) return response.json(); + if (!Number.isSafeInteger(maximum) || maximum < 1) + throw new Error('Invalid embedding response byte limit'); + const reader = response.body?.getReader(); + if (!reader) throw new Error('Embedding response has no body'); + let complete = false; + try { + const declared = response.headers.get('content-length'); + if (declared !== null && Number(declared) > maximum) + throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget'); + const bytes = Buffer.allocUnsafe(maximum); + let size = 0; + for (;;) { + const next = await reader.read(); + if (next.done) break; + if (next.value.byteLength > maximum - size) + throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget'); + bytes.set(next.value, size); + size += next.value.byteLength; + } + complete = true; + return JSON.parse(new TextDecoder().decode(bytes.subarray(0, size))); + } finally { + if (!complete) await reader.cancel().catch(() => {}); + reader.releaseLock(); + } } let lastHttpRequestStartedAt: number | undefined; @@ -427,6 +467,11 @@ const httpEmbedBatch = async ( timeoutMs = DEFAULT_HTTP_TIMEOUT_MS, retryTimeouts = false, ): Promise => { + if ( + requestOptions.maxResponseBytes !== undefined && + (!Number.isSafeInteger(requestOptions.maxResponseBytes) || requestOptions.maxResponseBytes < 1) + ) + throw new Error('Invalid embedding response byte limit'); const requestBody: { input: string[]; model: string; dimensions?: number } = { input: batch, model, @@ -490,8 +535,17 @@ const httpEmbedBatch = async ( // and the same breaker accounting as any other endpoint fault (#2790). let payload: { data: EmbeddingItem[] }; try { - payload = (await attemptResp.json()) as { data: EmbeddingItem[] }; + payload = (await readEmbeddingResponse( + attemptResp, + requestOptions.maxResponseBytes, + )) as { + data: EmbeddingItem[]; + }; } catch (err) { + if (err instanceof EmbeddingResponseBudgetError) + throw new EmbeddingResponseBudgetError( + `Embedding endpoint exceeded its ${requestOptions.maxResponseBytes}-byte response budget (${safeUrl(url)}, batch ${batchIndex})`, + ); // Not every `.json()` rejection is a parse error: the per-attempt // signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is // wired to the body stream, so a stalled body rejects with the abort @@ -540,6 +594,9 @@ const httpEmbedBatch = async ( }, ); } catch (err) { + if (err instanceof EmbeddingResponseBudgetError) { + throw new HttpEmbeddingError(err.message, { cause: err }); + } if ( requestOptions.signal?.aborted || (err instanceof DOMException && err.name === 'AbortError') diff --git a/gitnexus/src/core/embeddings/line-index.ts b/gitnexus/src/core/embeddings/line-index.ts index 2dc12de3d..265628e71 100644 --- a/gitnexus/src/core/embeddings/line-index.ts +++ b/gitnexus/src/core/embeddings/line-index.ts @@ -4,11 +4,15 @@ export interface ResolvedLineRange { } export const buildLineIndex = (content: string): Int32Array => { - const offsets: number[] = [0]; - for (let i = 0; i < content.length; i++) { - if (content.charCodeAt(i) === 10) offsets.push(i + 1); - } - return new Int32Array(offsets); + // Allocate the final packed index directly; a temporary number[] costs much + // more than the source for files with millions of short lines. + let count = 1; + for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) count++; + const offsets = new Int32Array(count); + let line = 1; + for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) + offsets[line++] = i + 1; + return offsets; }; const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => { diff --git a/gitnexus/src/core/embeddings/source-reader.ts b/gitnexus/src/core/embeddings/source-reader.ts index 26344214f..4b71569f7 100644 --- a/gitnexus/src/core/embeddings/source-reader.ts +++ b/gitnexus/src/core/embeddings/source-reader.ts @@ -1,7 +1,8 @@ import { createHash } from 'node:crypto'; -import { readFile, realpath } from 'node:fs/promises'; +import { realpath } from 'node:fs/promises'; import path from 'node:path'; import type { ContentRetention } from '../../storage/repo-meta.js'; +import { openEmbeddingJobFile } from '../../storage/embedding-job.js'; import { isNotebookPath, notebookPythonSnippet } from '../ingestion/ipynb-extractor.js'; import { buildLineIndex } from './line-index.js'; import { @@ -47,6 +48,7 @@ interface SourceFile { content: string; lines: Int32Array; bytes: number; + sourceBytes: number; } const CACHE_BYTES = 16 * 1024 * 1024; @@ -58,9 +60,17 @@ export class EmbeddingSourceReader { private bytes = 0; private root: Promise | undefined; - constructor(private readonly source: EmbeddingSourceContext) {} + constructor( + private readonly source: EmbeddingSourceContext, + private readonly cacheBytes = CACHE_BYTES, + ) {} - async read(node: EmbeddableNode): Promise { + /** Admission runs before allocation, including on cache hits. The caller + * holds its reservation until it finishes using/parsing the returned text. */ + async read( + node: EmbeddableNode, + admit?: (sourceBytes: number) => Promise, + ): Promise { if (this.source.contentRetention !== 'full') { throw new Error('Original embedding source requires full content retention'); } @@ -84,6 +94,7 @@ export class EmbeddingSourceReader { let file = this.files.get(node.filePath); if (file) { + await admit?.(file.sourceBytes); this.files.delete(node.filePath); this.files.set(node.filePath, file); } else { @@ -107,15 +118,38 @@ export class EmbeddingSourceReader { ) { throw new Error('source path leaves the repository'); } - const buffer = await readFile(resolved); + const { handle, stat } = await openEmbeddingJobFile(resolved); + let buffer: Buffer; + try { + if (!Number.isSafeInteger(stat.size) || stat.size < 0) + throw new Error('invalid source file size'); + await admit?.(stat.size); + // Bound the read to the admitted size, plus one byte to detect growth. + buffer = Buffer.alloc(stat.size + 1); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset); + if (!bytesRead) break; + offset += bytesRead; + } + if (offset !== stat.size) throw new Error('source size changed during the read'); + buffer = buffer.subarray(0, offset); + } finally { + await handle.close(); + } if (createHash('sha256').update(buffer).digest('hex') !== expectedHash) { throw new Error('checkout content differs from the indexed snapshot'); } const content = buffer.toString('utf8'); const lines = buildLineIndex(content); - file = { content, lines, bytes: content.length * 2 + lines.byteLength }; - if (file.bytes <= CACHE_BYTES) { - while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > CACHE_BYTES) { + file = { + content, + lines, + bytes: content.length * 2 + lines.byteLength, + sourceBytes: buffer.length, + }; + if (file.bytes <= this.cacheBytes) { + while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > this.cacheBytes) { const entry = this.files.entries().next().value; if (!entry) break; const [oldest, cached] = entry; diff --git a/gitnexus/src/core/embeddings/structural-extractor.ts b/gitnexus/src/core/embeddings/structural-extractor.ts index a5799cfb4..85d949d9f 100644 --- a/gitnexus/src/core/embeddings/structural-extractor.ts +++ b/gitnexus/src/core/embeddings/structural-extractor.ts @@ -5,11 +5,9 @@ * to extract method and field names for embedding text generation. */ -import { getProviderForFileContent } from '../ingestion/languages/index.js'; import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js'; import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js'; import type { LanguageProvider } from '../ingestion/language-provider.js'; -import { buildTypeEnv } from '../ingestion/type-env.js'; import { SupportedLanguages } from 'gitnexus-shared'; import { ensureAndParse, findDeclarationNode } from './ast-utils.js'; @@ -34,6 +32,7 @@ export const extractStructuralNames = async ( parsedTree?: Awaited>, expectedName?: string, ): Promise => { + const { getProviderForFileContent } = await import('../ingestion/languages/index.js'); const provider = getProviderForFileContent(filePath, content); if (!provider) return { methodNames: [], fieldNames: [] }; @@ -47,7 +46,7 @@ export const extractStructuralNames = async ( const language = provider.id; const methodNames = extractMethodNames(classNode, provider, filePath, language); - const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language); + const fieldNames = await extractFieldNames(classNode, provider, tree, filePath, language); return { methodNames, fieldNames }; }; @@ -67,15 +66,16 @@ function extractMethodNames( return result.methods.map((m) => m.name); } -function extractFieldNames( +async function extractFieldNames( classNode: any, provider: LanguageProvider, tree: any, filePath: string, language: SupportedLanguages, -): string[] { +): Promise { if (!provider.fieldExtractor) return []; + const { buildTypeEnv } = await import('../ingestion/type-env.js'); const typeEnv = buildTypeEnv(tree, language); const context: FieldExtractorContext = { diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index d5ab3f403..8af6ee5de 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -48,6 +48,8 @@ const truncateDescription = (text: string, maxLength: number): string => { * Clean code content for embedding */ const cleanContent = (content: string): string => { + // Keep this hot function small enough for V8 to inline ordinary snippets. + if (content.length >= 16 * 1024) return cleanLargeContent(content); return content .replace(/\r\n/g, '\n') .replace(/\n{3,}/g, '\n\n') @@ -57,6 +59,46 @@ const cleanContent = (content: string): string => { .trim(); }; +const cleanLargeContent = (content: string): string => { + const normalized = content.replace(/\r\n/g, '\n').replace(/\n{3,}/g, '\n\n'); + const output = new TextBlocks(); + for (const line of contentLines(normalized)) output.append(line.trimEnd()); + return output.finish().trim(); +}; + +/** Output-sized storage with a bounded number of temporary line strings. */ +class TextBlocks { + private pending: string[] = []; + private output = ''; + count = 0; + append(line: string): void { + if (this.count++) this.pending.push('\n'); + this.pending.push(line); + if (this.pending.length >= 1024) this.flush(); + } + private flush(): void { + this.output += this.pending.join(''); + this.pending = []; + } + finish(): string { + this.flush(); + return this.output; + } +} + +function* contentLines(content: string): Generator { + let start = 0; + for (;;) { + const end = content.indexOf('\n', start); + if (end < 0) { + yield content.slice(start); + return; + } + yield content.slice(start, end); + start = end + 1; + } +} + /** * Compact location signal for the embedding header: the last 1-2 path segments * (immediate parent dir + basename), never the full deep path. @@ -254,25 +296,34 @@ const DECL_START_RE = * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction). */ export const extractDeclarationOnly = (content: string): string => { - const lines = content.split('\n'); - const declLines: string[] = []; + const declLines = new TextBlocks(); let depth = 0; let started = false; let classDepth = 0; let skipDepth = 0; - for (const [idx, line] of lines.entries()) { + let nextLineStart = 0; + for (const line of contentLines(content)) { + nextLineStart += line.length + 1; const trimmed = line.trim(); if (!started) { if (DECL_START_RE.test(trimmed)) { // Non-brace language check: current line or next 3 lines must have `{` - const nextLines = lines.slice(idx + 1, idx + 4); - if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) { - return ''; + if (!trimmed.includes('{')) { + let lookaheadEnd = nextLineStart; + for (let i = 0; i < 3; i++) { + const end = content.indexOf('\n', lookaheadEnd); + if (end < 0) { + lookaheadEnd = content.length; + break; + } + lookaheadEnd = end + 1; + } + if (!content.slice(nextLineStart, lookaheadEnd).includes('{')) return ''; } started = true; - declLines.push(trimmed); + declLines.append(trimmed); for (const ch of trimmed) { if (ch === '{') depth++; else if (ch === '}') depth--; @@ -283,8 +334,12 @@ export const extractDeclarationOnly = (content: string): string => { } // Always update depth (even when skipping) - const opens = (trimmed.match(/{/g) || []).length; - const closes = (trimmed.match(/}/g) || []).length; + let opens = 0; + let closes = 0; + for (let i = 0; i < trimmed.length; i++) { + if (trimmed.charCodeAt(i) === 123) opens++; + else if (trimmed.charCodeAt(i) === 125) closes++; + } const prevDepth = depth; depth += opens - closes; @@ -293,7 +348,7 @@ export const extractDeclarationOnly = (content: string): string => { skipDepth = 0; // Closing brace of class if (depth <= 0) { - declLines.push(trimmed); + declLines.append(trimmed); break; } } @@ -306,7 +361,7 @@ export const extractDeclarationOnly = (content: string): string => { if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) { if (opens === closes && trimmed.endsWith(';')) { // Property with function/object initializer like `config = { timeout: 5000 };` — keep - declLines.push(trimmed); + declLines.append(trimmed); } // else: single-line or multi-line method — skip entirely if (opens !== closes) { @@ -315,12 +370,12 @@ export const extractDeclarationOnly = (content: string): string => { continue; } - declLines.push(trimmed); + declLines.append(trimmed); - if (depth <= 0 && declLines.length > 1) break; + if (depth <= 0 && declLines.count > 1) break; } - return declLines.join('\n').trim(); + return declLines.finish().trim(); }; /** diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 65a4fd5f6..6ed1a8408 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -3,6 +3,7 @@ * * Type definitions for the embedding generation and semantic search system. */ +import type { AnalysisPhaseProgress } from '../analysis-progress.js'; export const LABEL_FUNCTION = 'Function' as const; export const LABEL_METHOD = 'Method' as const; @@ -206,7 +207,7 @@ export type EmbeddingPhase = /** * Progress information for the embedding pipeline */ -export interface EmbeddingProgress { +export interface EmbeddingProgress extends AnalysisPhaseProgress { phase: EmbeddingPhase; percent: number; modelDownloadPercent?: number; diff --git a/gitnexus/src/core/embeddings/vector-generation-search.ts b/gitnexus/src/core/embeddings/vector-generation-search.ts new file mode 100644 index 000000000..2b1b25ef8 --- /dev/null +++ b/gitnexus/src/core/embeddings/vector-generation-search.ts @@ -0,0 +1,138 @@ +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { createHash } from 'node:crypto'; +import type { EmbeddingIdentity } from './embedding-identity.js'; +import { + assertOwnedVectorGeneration, + selectVectorGeneration, +} from '../../storage/vector-generation.js'; +import { + initLbug, + pinRepo, + executeParameterized, + ensureVectorExtension, +} from '../lbug/pool-adapter.js'; +import { VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js'; +import { getExactScanLimit } from '../platform/capabilities.js'; +import { collectBestChunks } from './types.js'; +import { rankExactEmbeddingRows, type ExactSearchChunk } from './exact-search.js'; +import { assertQueryIdentity, rethrowSymbolIdentityError } from '../lbug/query-result-integrity.js'; +import { logger } from '../logger.js'; + +export interface VectorSearchSnapshot { + metadataDirectory: string; + metadata: RepoMeta | null; + graphPath: string; +} + +const configuredIdentity = async (): Promise => { + const { resolveEmbeddingIdentity } = await import('./embedding-identity.js'); + return resolveEmbeddingIdentity(); +}; + +/** null means a legacy monolithic reader is required. An empty array is a + * deliberate unavailable lane and must not fall through to legacy vectors. */ +export async function searchVectorGeneration( + snapshot: VectorSearchSnapshot, + embed: () => Promise, + limit: number, + maxDistance: number, + resolveIdentity: () => Promise = configuredIdentity, +): Promise { + const exactLimit = getExactScanLimit(); + const selected = selectVectorGeneration( + snapshot.metadataDirectory, + snapshot.metadata, + snapshot.graphPath, + exactLimit, + ); + if (selected.kind === 'legacy') return null; + if (selected.kind !== 'ready') return []; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000 || !Number.isFinite(maxDistance)) + throw new Error('Invalid vector search bounds'); + const dbPath = selected.path; + const artifactIdentity = selected.generation.artifactIdentity ?? selected.generation.identity; + const release = pinRepo(dbPath); + try { + await assertOwnedVectorGeneration(snapshot.metadataDirectory, artifactIdentity); + await initLbug(dbPath, dbPath, { + kind: 'vector', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 2, + }); + const execute = (query: string, parameters: Record = {}) => + executeParameterized(dbPath, query, parameters); + const artifact = await execute( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.identity AS identity, a.schema AS schema, a.dimensions AS dimensions, a.complete AS complete, a.indexKind AS kind", + ); + const proof = artifact[0]; + if ( + artifact.length !== 1 || + proof.identity !== artifactIdentity || + proof.schema !== VECTOR_STORE_SCHEMA || + Number(proof.dimensions) !== selected.generation.dimensions || + proof.complete !== true || + proof.kind !== selected.generation.indexKind + ) + throw new Error('Published vector identity is unverified'); + const annReady = + selected.generation.indexKind === 'hnsw' && + (await ensureVectorExtension(dbPath).catch((error: unknown) => { + logger.warn( + { err: error }, + 'Published VECTOR extension could not load; checking the bounded exact-search fallback', + ); + return false; + })); + if (!annReady && selected.generation.chunks > exactLimit) return []; + const identity = await resolveIdentity(); + const vectorIdentity = createHash('sha256') + .update(JSON.stringify([identity.provider, identity.model, identity.dimensions])) + .digest('hex'); + if (vectorIdentity !== selected.generation.vectorIdentity) return []; + // Neither a metadata pointer nor a timestamp alone authorizes a model call. + const query = await embed(); + if ( + query.length !== selected.generation.dimensions || + query.some((value) => !Number.isFinite(value)) + ) + throw new Error('Query embedding does not match the published vector dimensions'); + if (annReady) { + try { + const best = await collectBestChunks(limit, async (fetchLimit) => { + const rows = await execute( + `CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', $vector, $limit) + YIELD node AS emb, distance WITH emb, distance WHERE distance < $distance + RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, emb.startLine AS startLine, emb.endLine AS endLine, distance ORDER BY distance`, + { vector: [...query], limit: fetchLimit, distance: maxDistance }, + ); + return rows.map((row) => { + assertQueryIdentity(row, 'nodeId', 0); + return row as ExactSearchChunk; + }); + }); + if (best.size > 0) return [...best].map(([nodeId, chunk]) => ({ nodeId, ...chunk })); + } catch (error) { + rethrowSymbolIdentityError(error); + if (selected.generation.chunks > exactLimit) throw error; + } + } + if (selected.generation.chunks > exactLimit) return []; + // determinism: probe — fetch the entire verified set; reject partial or oversized coverage before ranking. + const rows = + await execute(`MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, + e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding LIMIT ${exactLimit + 1}`); + if (rows.length !== selected.generation.chunks || rows.length > exactLimit) + throw new Error('Published vector coverage does not match its receipt'); + for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); + const seen = new Set(); + return rankExactEmbeddingRows(rows, query, rows.length, maxDistance) + .filter((row) => { + if (seen.has(row.nodeId)) return false; + seen.add(row.nodeId); + return true; + }) + .slice(0, limit); + } finally { + release(); + } +} diff --git a/gitnexus/src/core/embeddings/vector-publication.ts b/gitnexus/src/core/embeddings/vector-publication.ts new file mode 100644 index 000000000..709979b0d --- /dev/null +++ b/gitnexus/src/core/embeddings/vector-publication.ts @@ -0,0 +1,108 @@ +import path from 'node:path'; +import type { RepoMeta } from '../../storage/repo-meta.js'; +import { + assertOwnedVectorGeneration, + selectVectorGeneration, + type PublishedVectorGeneration, +} from '../../storage/vector-generation.js'; +import { VectorStore } from '../lbug/vector-store.js'; +import { syncPublishedMetadata } from '../analysis-publication.js'; +import { getExactScanLimit } from '../platform/capabilities.js'; +import { reclaimAnalysisArtifacts } from '../../storage/analysis-artifact-cleanup.js'; +import { readEmbeddingArtifactOwners } from '../../storage/embedding-artifact.js'; + +export interface VectorPublicationActions { + loadMetadata: () => Promise; + saveMetadata: (metadata: RepoMeta) => Promise; +} + +/** Caller holds the checkout's index lock and has closed the vector writer. + * A failed verify/close/save leaves the job in indexing, with no paid replay. + * null means ANN remains pending beyond the exact-scan policy limit. + */ +export async function publishVectorGeneration( + metadataDirectory: string, + generation: PublishedVectorGeneration, + resources: { bufferPoolBytes: number; threads: number; batchBytes: number }, + actions: VectorPublicationActions, + migratedLegacySource?: string, +): Promise { + const exactScanLimit = getExactScanLimit(); + if (generation.indexKind === 'exact' && generation.chunks > exactScanLimit) return null; + const previous = await actions.loadMetadata(); + if ( + !previous?.graphGeneration || + previous.incrementalInProgress || + previous.graphGeneration.id !== generation.graphGeneration || + previous.embeddingJob?.identity !== generation.identity || + previous.embeddingJob?.graphGeneration !== generation.graphGeneration || + (previous.embeddingJob.artifactIdentity ?? previous.embeddingJob.identity) !== + (generation.artifactIdentity ?? generation.identity) + ) + throw new Error('Vector publication does not match the active graph job'); + const metadata: RepoMeta = { + ...previous, + vectorGeneration: generation, + embeddingJob: undefined, + embeddingCheckpoint: undefined, + embeddingDims: generation.dimensions, + stats: { ...previous.stats, embeddings: generation.chunks }, + capabilities: { + graph: previous.capabilities?.graph ?? { provider: 'ladybugdb', status: 'available' }, + fts: previous.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' }, + vectorSearch: { + provider: generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: generation.chunks + ? generation.indexKind === 'hnsw' + ? 'vector-index' + : 'exact-scan' + : 'unavailable', + exactScanLimit, + ...(generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}), + }, + }, + }; + const selection = selectVectorGeneration( + metadataDirectory, + metadata, + path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'), + exactScanLimit, + ); + if ( + selection.kind !== 'ready' && + !(selection.kind === 'unavailable' && selection.reason === 'embeddings-empty') + ) + throw new Error('Invalid vector publication receipt'); + const artifactIdentity = generation.artifactIdentity ?? generation.identity; + const file = await assertOwnedVectorGeneration(metadataDirectory, artifactIdentity); + const store = await VectorStore.open({ + directory: path.dirname(file), + identity: artifactIdentity, + dimensions: generation.dimensions, + mode: 'read', + ...resources, + }); + try { + if ( + (await store.seal(generation.nodes - (generation.excludedNodes ?? 0), generation.chunks)) !== + generation.indexKind + ) + throw new Error('Published vector index does not match the native artifact'); + } finally { + await store.close(); + } + if (previous.legacyVectorSource && previous.legacyVectorSource === migratedLegacySource) { + const owners = await readEmbeddingArtifactOwners(metadataDirectory); + if ( + owners.some( + (owner) => owner.artifactIdentity === artifactIdentity && owner.phase === 'sealed', + ) && + !owners.some((owner) => owner.fallbackLegacy === previous.legacyVectorSource) + ) + delete metadata.legacyVectorSource; + } + await actions.saveMetadata(metadata); + await syncPublishedMetadata(metadataDirectory, metadata); + await reclaimAnalysisArtifacts(metadataDirectory, metadata); + return metadata; +} diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index 52e62e8d8..6ae7351a4 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -6,6 +6,7 @@ export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', 'embedding-count-unverified', + 'embedding-job-pending', 'graph-write-collapsed', 'scope-extraction-unverified', 'scope-extraction-failed', @@ -137,6 +138,7 @@ export function getIndexIncompleteReasons( RepoMeta, | 'incrementalInProgress' | 'embeddingCheckpoint' + | 'embeddingJob' | 'graphWriteCollapsed' | 'scopeExtractionFailures' | 'scopeExtractionReceipt' @@ -172,5 +174,18 @@ export function getIndexIncompleteReasons( : 'embedding-checkpoint-pending', ); } + if (meta?.embeddingJob) reasons.push('embedding-job-pending'); return reasons; } + +/** Embedding debt does not invalidate a published graph or require parsing. */ +export function getGraphIncompleteReasons( + meta: Parameters[0], +): IndexIncompleteReason[] { + return getIndexIncompleteReasons(meta).filter( + (reason) => + reason !== 'embedding-checkpoint-pending' && + reason !== 'embedding-count-unverified' && + reason !== 'embedding-job-pending', + ); +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index a93a4b0a7..9d5879cf7 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -616,10 +616,23 @@ export async function runChunkedParseAndResolve( // shrink an incremental re-analyze: `totalParseable` counts every parseable // file in the scan, not the changed ones, so a warm run of a large repo still // spawns the full requested pool. - const effectivePoolSize = + const workerResources = options?.workerResources; + if ( + totalParseable > 0 && + workerResources && + (!Number.isSafeInteger(workerResources.workers) || + workerResources.workers < 1 || + !Number.isSafeInteger(workerResources.workerHeapMb) || + workerResources.workerHeapMb < 512 || + (explicitPoolSize !== undefined && explicitPoolSize > workerResources.workers)) + ) + throw new Error('Parser worker configuration exceeds its admitted resource budget'); + const effectivePoolSize = Math.min( explicitPoolSize && explicitPoolSize > 0 ? Math.min(explicitPoolSize, Math.max(1, totalParseable)) - : Math.min(resolveAutoPoolSize(), workProportionalCap); + : Math.min(resolveAutoPoolSize(), workProportionalCap), + workerResources?.workers ?? Infinity, + ); // Deliberate over-subscription is the operator's call, so this warns rather // than caps — silently capping is what the override exists to stop. But an // exported `GITNEXUS_WORKER_POOL_SIZE` applies to EVERY analyze in a @@ -725,6 +738,9 @@ export async function runChunkedParseAndResolve( // after it is initialized; `undefined` drives the worker no-store // fallback (return ParsedFiles in the result). workerPool = createWorkerPool(workerUrl, effectivePoolSize, { + ...(workerResources + ? { heapLimitMb: workerResources.workerHeapMb, requireDrain: true } + : {}), parsedFileStoreStoragePath: parsedFileStorePath, // Durable, content-addressed shard dir for warm-cache reuse (#2038). // Initialized below before the chunk loop (same deferred-init pattern @@ -817,6 +833,8 @@ export async function runChunkedParseAndResolve( let chunkCacheMisses = 0; let parseCacheHitFileCount = 0; let reparsedFileCount = 0; + let parseFailure: unknown; + let parsingFailed = false; try { // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch @@ -1227,6 +1245,7 @@ export async function runChunkedParseAndResolve( } let missIdx = 0; for (const entry of round.entries) { + options?.signal?.throwIfAborted(); if (entry.kind === 'hit') { const chunkWorkerData = mergeChunkResults( graph, @@ -1257,6 +1276,7 @@ export async function runChunkedParseAndResolve( * park it resolved for the next close to merge. */ const closeRound = async (): Promise => { + options?.signal?.throwIfAborted(); const started = await startRound(roundEntries); roundEntries = []; roundBudget.reset(); @@ -1291,6 +1311,7 @@ export async function runChunkedParseAndResolve( }; for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + options?.signal?.throwIfAborted(); if (heapProbeEveryN > 0 && chunkIdx > 0 && chunkIdx % heapProbeEveryN === 0) { logHeapProbe( `parse-chunk-${chunkIdx}`, @@ -1525,8 +1546,23 @@ export async function runChunkedParseAndResolve( `processRoutesFromExtracted: ${ms.toFixed(0)}ms (${allExtractedRoutes.length} routes)`, ); } + } catch (error) { + parseFailure = error; + parsingFailed = true; + throw error; } finally { - await workerPool?.terminate(); + try { + await workerPool?.terminate(); + } catch (drainError) { + if (parsingFailed) { + throw new AggregateError( + [parseFailure, drainError], + `Parsing failed (${parseFailure instanceof Error ? parseFailure.message : String(parseFailure)}) and native workers have not drained`, + { cause: parseFailure }, + ); + } + throw drainError; + } } // Fetch calls + ORM queries were already extracted inside each worker diff --git a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts index da8e4dd8f..79e4da9f5 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts @@ -203,6 +203,7 @@ export async function runPipeline( const results = new Map>(seed); for (const phase of sorted) { + ctx.options?.signal?.throwIfAborted(); const start = Date.now(); if (isDev) { diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 9ec3c5981..486c67c9a 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -56,6 +56,8 @@ import { } from './pipeline-phases/index.js'; export interface PipelineOptions { + /** Cooperative cancellation checked between parser dispatch and graph phases. */ + signal?: AbortSignal; /** * Skip MRO, community detection, and process extraction for faster test runs. * The `pruneLocalSymbols` phase still runs — it is graph construction (it cleans @@ -255,6 +257,8 @@ export interface PipelineOptions { * analyze invocations. */ workerPoolSize?: number; + /** Admitted ceiling, separate from an explicit worker count so tiny repos retain auto-sizing. */ + workerResources?: Readonly<{ workers: number; workerHeapMb: number }>; /** * Process-detection budget (#3313). Explicit `maxProcesses` replaces the * dynamic `symbols / 10` formula; the other three replace compiled defaults. @@ -388,6 +392,7 @@ export const runPipelineFromRepo = async ( onProgress: (progress: PipelineProgress) => void, options?: PipelineOptions, ): Promise => { + options?.signal?.throwIfAborted(); const graph = createKnowledgeGraph(); const pipelineStart = Date.now(); @@ -427,6 +432,7 @@ export const runPipelineFromRepo = async ( let results; try { results = await runPipeline(phases, ctx); + options?.signal?.throwIfAborted(); graphEmitManifest = graphEmitSink?.finalize(); } finally { // Release per-pair fds when the pipeline threw before finalize ran. diff --git a/gitnexus/src/core/ingestion/utils/effective-ram.ts b/gitnexus/src/core/ingestion/utils/effective-ram.ts index b5ea977c6..78aacc1ec 100644 --- a/gitnexus/src/core/ingestion/utils/effective-ram.ts +++ b/gitnexus/src/core/ingestion/utils/effective-ram.ts @@ -99,7 +99,7 @@ export function heapPressureRemedy( if (heapLimitBytes < autoCapBytes * 0.9) { if (source === 'budget') { return ( - `This machine has more memory available: raise --memory-budget, or omit it ` + + `This machine has more memory available: raise --memory-budget (and --resource-budget if set), or omit the limits ` + `so gitnexus sizes its heap to the machine automatically.` ); } diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index b972d7326..bd59b87ec 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -220,6 +220,10 @@ export interface WorkerPoolStats { } export interface WorkerPoolOptions { + /** Per-run old-generation allocation; overrides the process-wide environment default. */ + heapLimitMb?: number; + /** Refuse to release the parser reservation while a retired native worker still runs. */ + requireDrain?: boolean; subBatchSize?: number; subBatchMaxBytes?: number; subBatchIdleTimeoutMs?: number; @@ -1053,6 +1057,7 @@ export const createWorkerPool = ( const size = poolSize ?? resolveAutoPoolSize(); const poolOptions = resolveWorkerPoolOptions(options, size); + const requireDrain = options?.requireDrain === true; // Production factory spawns with `{ stderr: true }` so a worker's crash // output is redirected to a `worker.stderr` stream we can tee + capture // (see captureWorkerStderr) and attach to readiness-failure messages — @@ -1071,7 +1076,12 @@ export const createWorkerPool = ( parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg ? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines } : undefined; - const workerHeapCapMb = resolveWorkerHeapCapMb(size); + const workerHeapCapMb = options?.heapLimitMb ?? resolveWorkerHeapCapMb(size); + if ( + options?.heapLimitMb !== undefined && + (!Number.isSafeInteger(workerHeapCapMb) || workerHeapCapMb < 512) + ) + throw new Error('Parser worker heap allocation must be a safe integer of at least 512 MiB'); // The 512MB per-worker floor exists so a worker can parse anything real, // but on a very small container a large pool of floored workers can still // overcommit total memory (#2649 review). Behavior is unchanged — deaths @@ -1135,6 +1145,7 @@ export const createWorkerPool = ( safePoint: Promise; }; const retiredWorkers = new Set(); + const pendingWorkerReleases = new Map, number>(); const respawnCount: number[] = new Array(size).fill(0); const activeSlots: Set = new Set(); // Layer 3 (quarantine): tracked via the dedicated `quarantine.ts` @@ -1184,7 +1195,31 @@ export const createWorkerPool = ( ): Promise => { const retired = Array.from(retiredWorkers); await Promise.all([ - ...liveWorkers.map((worker) => worker?.terminate().catch(() => undefined)), + ...liveWorkers.map(async (worker, workerIndex) => { + if (!worker) return; + let completeRelease!: () => void; + const termination = new Promise((resolve) => { + completeRelease = resolve; + }); + const released = () => { + worker.off('exit', released); + pendingWorkerReleases.delete(termination); + completeRelease(); + }; + if (requireDrain) { + pendingWorkerReleases.set(termination, workerIndex); + worker.once('exit', released); + } + try { + await worker.terminate(); + released(); + } catch { + // A rejected request does not prove physical exit. Retain ownership + // after workers[] is cleared, and release it on the eventual exit. + if (!requireDrain) released(); + else worker.unref?.(); + } + }), ...retired.map(async (record) => { // #2432: a retired worker that has not reached a JS-visible safe // point may be inside an N-API call — terminating it aborts the @@ -1492,14 +1527,14 @@ export const createWorkerPool = ( reason: string, ): void => { let cleaned = false; - let terminateStarted = false; + let termination: Promise | undefined; let resolveSafePoint!: () => void; const safePoint = new Promise((resolve) => { resolveSafePoint = resolve; }); // A message/messageerror proves the worker is executing JS again; an - // exit/error means the thread is gone. Either way `worker.terminate()` + // exit/error means native execution has stopped. Either way `worker.terminate()` // can no longer land mid-N-API call (#2432), so shutdown's drain may // stop waiting. function markSafeToTerminate() { @@ -1517,11 +1552,14 @@ export const createWorkerPool = ( retiredWorkers.delete(record); } - async function terminateRetired() { - if (terminateStarted) return; - terminateStarted = true; - cleanupRetired(); - await worker.terminate().catch(() => undefined); + function terminateRetired(): Promise { + termination ??= worker.terminate().then( + () => cleanupRetired(), + () => { + if (!requireDrain) cleanupRetired(); + }, + ); + return termination; } function terminateWhenBackInJs() { @@ -1539,7 +1577,9 @@ export const createWorkerPool = ( const onRetiredError = () => { markSafeToTerminate(); - cleanupRetired(); + // The error event precedes physical exit. Keep the reservation and + // exit listener if termination subsequently rejects or stalls. + if (!requireDrain) cleanupRetired(); }; const onRetiredExit = () => { markSafeToTerminate(); @@ -1581,10 +1621,24 @@ export const createWorkerPool = ( } // Recovery must settle before dispatch returns, but a failed thread // may never acknowledge termination. Bound that wait as in shutdown. - const termination = existing.terminate().then( - () => undefined, - () => undefined, - ); + let completeRelease!: () => void; + const termination = new Promise((resolve) => { + completeRelease = resolve; + }); + const released = () => { + existing.off('exit', released); + pendingWorkerReleases.delete(termination); + completeRelease(); + }; + if (requireDrain) { + pendingWorkerReleases.set(termination, workerIndex); + // A rejected terminate request is not proof of release. An eventual + // physical exit is, even when the termination promise rejected. + existing.once('exit', released); + } + void existing.terminate().then(released, () => { + if (!requireDrain) released(); + }); if (!(await settledWithin(termination, poolOptions.shutdownDrainMs))) { existing.unref?.(); logger.warn( @@ -1601,6 +1655,22 @@ export const createWorkerPool = ( ): Promise => { await removeWorkerFromSlot(workerIndex, mode, reason); if (stopped) return false; + if (requireDrain) { + // A timeout retires a native-active worker without killing it. Its + // slot remains reserved until both the safe point and native exit. + for (const record of retiredWorkers) { + if (record.workerIndex !== workerIndex) continue; + if ( + !record.safeToTerminate && + !(await settledWithin(record.safePoint, poolOptions.shutdownDrainMs)) + ) + return false; + if (!(await settledWithin(record.terminate(), poolOptions.shutdownDrainMs))) + return false; + if (retiredWorkers.has(record)) return false; + } + if ([...pendingWorkerReleases.values()].includes(workerIndex) || stopped) return false; + } const replacement = spawnAndCapture(workerUrl); try { await waitForWorkerReady(replacement, poolOptions.workerReadyTimeoutMs); @@ -2418,6 +2488,12 @@ export const createWorkerPool = ( await terminateTrackedWorkers(workers); workers.length = 0; activeSlots.clear(); + if (requireDrain && (retiredWorkers.size > 0 || pendingWorkerReleases.size > 0)) + throw new Error( + 'Parser workers have not released their native resource reservation; refusing to start another analysis phase. ' + + `Suspect files: ${quarantine.snapshot().slice(0, 5).join(', ') || 'see the last parser progress message'}. ` + + 'Workers will exit at their next safe point. Exclude a repeatedly stuck file with .gitnexusignore and retry; do not kill a native worker thread.', + ); }; const dispatch = async ( diff --git a/gitnexus/src/core/lbug/embedding-source.ts b/gitnexus/src/core/lbug/embedding-source.ts new file mode 100644 index 000000000..67b2e15b6 --- /dev/null +++ b/gitnexus/src/core/lbug/embedding-source.ts @@ -0,0 +1,198 @@ +/** Key-only manifest scans and bounded PK fetches against one immutable graph. */ +import lbug from '@ladybugdb/core'; +import { + closeLbugConnection, + openLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; +import { + EMBEDDABLE_LABELS, + LABELS_WITH_EXPORTED, + LABEL_METHOD, + type EmbeddableNode, +} from '../embeddings/types.js'; +import type { + EmbeddingManifestSource, + EmbeddingWorkKey, +} from '../embeddings/embedding-work-manifest.js'; + +const LABELS = new Set([...EMBEDDABLE_LABELS, 'File']); +// ECMAScript trim's whitespace set, including Unicode separators and BOM. +// RE2's default \\S is ASCII-only and would change the File fallback selection. +const NON_WHITESPACE = + '[^\\x09-\\x0d\\x20\\x{a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]'; + +function checkedLabel(label: string): string { + if (!LABELS.has(label)) throw new Error('Unsupported embedding manifest label'); + return `\`${label}\``; +} + +function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult { + if (!Array.isArray(result)) return result; + if (result.length === 1) return result[0]; + for (const part of result) part.close(); + throw new Error('Unexpected multiple graph query results'); +} + +function size(value: unknown): number { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid graph byte count'); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) throw new Error('Invalid graph byte count'); + return result; +} + +function string(value: unknown): string { + if (typeof value !== 'string') throw new Error('Invalid graph text field'); + return value; +} +function optionalString(value: unknown): string | undefined { + return value == null ? undefined : string(value); +} + +export class EmbeddingGraphSource implements EmbeddingManifestSource { + private active = false; + private closed = false; + private closing?: Promise; + private readonly fetchStatements = new Map(); + private constructor( + private readonly handle: LbugConnectionHandle, + private readonly graphPath: string, + ) {} + + /** graphPath has already passed the publication layer's ownership validation. */ + static async open( + graphPath: string, + bufferPoolBytes: number, + threads: number, + ): Promise { + return new EmbeddingGraphSource( + await openLbugConnection(lbug, graphPath, { readOnly: true, bufferPoolBytes, threads }), + graphPath, + ); + } + + private acquire(): void { + if (this.closed || this.active) + throw new Error('Embedding graph source is closed or already in use'); + this.active = true; + } + + async *scan(label: string): AsyncGenerator { + const table = checkedLabel(label); + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + const fields = ['id', 'name', 'filePath']; + if (label !== 'File') fields.push('description'); + if (label === LABEL_METHOD) fields.push('returnType'); + // JSON escaping costs at most six bytes per source byte. The fixed + // allowance covers property names, numeric spans, booleans and the label. + const metadataBytes = fields + .map((field) => `octet_length(encode(coalesce(n.${field}, '')))`) + .join(' + '); + const filter = + label === 'File' + ? "WHERE n.filePath IS NOT NULL AND n.filePath <> '' AND n.content <> '[Binary file - content not stored]' AND regexp_matches(coalesce(n.content,''), $nonWhitespace)" + : ''; + const sql = `MATCH (n:${table}) ${filter} RETURN n.id AS id, octet_length(encode(coalesce(n.content,''))) AS contentBytes, 256 + 6 * (${metadataBytes}) AS metadataBytes ORDER BY n.id`; + if (label === 'File') { + const statement = await this.handle.conn.prepare(sql); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + result = single( + await this.handle.conn.execute(statement, { nonWhitespace: NON_WHITESPACE }), + ); + } else result = single(await this.handle.conn.query(sql)); + while (result.hasNext()) { + const row = result.getNextSync(); + yield { + id: string(row.id), + contentBytes: size(row.contentBytes), + metadataBytes: size(row.metadataBytes), + }; + } + } finally { + try { + result?.close(); + } finally { + this.active = false; + } + } + } + + async fetch(label: string, ids: readonly string[]): Promise { + const table = checkedLabel(label); + if (!ids.length || ids.length > 64 || new Set(ids).size !== ids.length) + throw new Error('Invalid graph fetch unit'); + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + let statement = this.fetchStatements.get(label); + if (!statement) { + const fields = ['id', 'name', 'filePath', 'content']; + if (label !== 'File') fields.push('startLine', 'endLine', 'description'); + if (LABELS_WITH_EXPORTED.has(label)) fields.push('isExported'); + if (label === LABEL_METHOD) fields.push('parameterCount', 'returnType'); + statement = await this.handle.conn.prepare( + `UNWIND $ids AS id MATCH (n:${table} {id:id}) RETURN ${fields.map((field) => `n.${field} AS ${field}`).join(', ')}`, + ); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + this.fetchStatements.set(label, statement); + } + result = single(await this.handle.conn.execute(statement, { ids: [...ids] })); + const nodes: EmbeddableNode[] = []; + while (result.hasNext()) { + if (nodes.length >= ids.length) throw new Error('Graph fetch exceeded its manifest unit'); + const row = result.getNextSync(); + const content = row.content ?? ''; + if (typeof content !== 'string') throw new Error('Graph content is not text'); + let fileLines = 1; + if (label === 'File') + for (let i = 0; i < content.length; i++) if (content.charCodeAt(i) === 10) fileLines++; + if (row.isExported != null && typeof row.isExported !== 'boolean') + throw new Error('Invalid graph export flag'); + nodes.push({ + id: string(row.id), + name: string(row.name), + label, + filePath: string(row.filePath), + content, + startLine: label === 'File' ? 1 : row.startLine == null ? undefined : size(row.startLine), + endLine: + label === 'File' ? fileLines : row.endLine == null ? undefined : size(row.endLine), + ...(label === 'File' ? {} : { description: optionalString(row.description) }), + ...(LABELS_WITH_EXPORTED.has(label) + ? { isExported: row.isExported == null ? undefined : (row.isExported as boolean) } + : {}), + ...(label === LABEL_METHOD + ? { + parameterCount: row.parameterCount == null ? undefined : size(row.parameterCount), + returnType: optionalString(row.returnType), + } + : {}), + }); + } + return nodes; + } finally { + try { + result?.close(); + } finally { + this.active = false; + } + } + } + + async close(): Promise { + if (this.closing) return this.closing; + if (this.active) throw new Error('Cannot close an active embedding graph read'); + this.closed = true; + this.fetchStatements.clear(); + this.closing = closeLbugConnection(this.handle, { strict: true, databasePath: this.graphPath }); + try { + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; + } + } +} diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 4aea794b1..dcd5bf424 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -59,6 +59,7 @@ import { resolveFtsVersionPair } from './vendored-extension-path.js'; import { classifyDeleteAllError, closeLbugConnection, + drainLbugResourceReleases, HANDLE_RELEASE_PROBE_ATTEMPTS, HANDLE_RELEASE_PROBE_DELAY_MS, isDbBusyError, @@ -624,6 +625,7 @@ const writableFtsCrashWalEvidence = async ( const reopenWritableAfterMissingShadow = async ( dbPath: string, err: unknown, + options: LbugInitOptions = {}, ): Promise => { // Analyze writers may park a leftover in-place FTS abort WAL. Serve/embed // refuse first via assertReadOnlyFtsCrashSafe and must not pass this @@ -644,7 +646,12 @@ const reopenWritableAfterMissingShadow = async ( throw new Error(renameFailureMessage(dbPath, renameErr)); } - return await openLbugConnection(lbug, dbPath); + return options.bufferPoolBytes !== undefined || options.threads !== undefined + ? await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }) + : await openLbugConnection(lbug, dbPath); }; const ensureReadOnlyConnectionUsable = async ( @@ -802,8 +809,18 @@ const runSchemaCreationQueries = async (dbPath: string): Promise return null; }; -export const initLbug = async (dbPath: string, options: { skipFts?: boolean } = {}) => { - return runWithSessionLock(() => ensureLbugInitialized(dbPath, options)); +export interface LbugInitOptions { + skipFts?: boolean; + bufferPoolBytes?: number; + threads?: number; +} + +export const initLbug = async (dbPath: string, options: LbugInitOptions = {}) => { + return runWithSessionLock(async () => { + if (options.bufferPoolBytes !== undefined || options.threads !== undefined) + await drainLbugResourceReleases(); + return ensureLbugInitialized(dbPath, options); + }); }; /** @@ -856,34 +873,51 @@ export const withLbugDb = async ( }; let currentDbSkipFts = false; +let currentDbBufferPoolBytes: number | undefined; +let currentDbThreads: number | undefined; +let nativeCloseUnverified = false; const ensureLbugInitialized = async ( dbPath: string, - options: { readOnly?: boolean; skipFts?: boolean } = {}, + options: LbugInitOptions & { readOnly?: boolean } = {}, ) => { const readOnly = options.readOnly === true; const skipFts = options.skipFts === true; + if ( + options.bufferPoolBytes !== undefined && + (!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes < 8 * 1024 * 1024) + ) + throw new Error('Explicit LadybugDB buffer pool must be at least 8 MiB'); + if ( + options.threads !== undefined && + (!Number.isSafeInteger(options.threads) || options.threads < 1) + ) + throw new Error('Explicit LadybugDB thread allocation must be a positive integer'); if ( conn && + !nativeCloseUnverified && currentDbPath === dbPath && currentDbReadOnly === readOnly && - currentDbSkipFts === skipFts + currentDbSkipFts === skipFts && + currentDbBufferPoolBytes === options.bufferPoolBytes && + currentDbThreads === options.threads ) { return { db, conn }; } - await doInitLbug(dbPath, { readOnly, skipFts }); + await doInitLbug(dbPath, { ...options, readOnly, skipFts }); return { db, conn }; }; const doInitLbug = async ( dbPath: string, - options: { readOnly?: boolean; skipFts?: boolean } = {}, + options: LbugInitOptions & { readOnly?: boolean } = {}, ) => { const readOnly = options.readOnly === true; const skipFts = options.skipFts === true; // Different database requested — close the old one first - if (conn || db) { - await safeClose(); + const resourceBound = options.bufferPoolBytes !== undefined || options.threads !== undefined; + if (conn || db || nativeCloseUnverified || (resourceBound && currentDbPath)) { + await safeClose({ strict: resourceBound }); currentDbPath = null; ftsLoaded = false; vectorExtensionLoaded = false; @@ -1045,10 +1079,19 @@ const doInitLbug = async ( }); try { - const opened = await openLbugConnection(lbug, dbPath); + const opened = resourceBound + ? await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }) + : await openLbugConnection(lbug, dbPath); db = opened.db; conn = opened.conn; currentDbReadOnly = false; + // Record the live allocation before schema work, so an initialization + // failure must also prove release before another native open. + currentDbBufferPoolBytes = options.bufferPoolBytes; + currentDbThreads = options.threads; } catch (err) { // Incremental analyze can hit a storage-version mismatch on construct // or (more often) on the first schema query below. Fail immediately @@ -1066,7 +1109,7 @@ const doInitLbug = async ( if (missingShadowError) { await safeClose(); resetOpenConnectionState(); - const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError); + const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError, options); db = reopened.db; conn = reopened.conn; currentDbReadOnly = false; @@ -1094,6 +1137,8 @@ const doInitLbug = async ( } currentDbSkipFts = skipFts; + currentDbBufferPoolBytes = options.bufferPoolBytes; + currentDbThreads = options.threads; currentDbPath = dbPath; return { db, conn }; }; @@ -2399,7 +2444,9 @@ export const fetchExistingEmbeddingHashes = async ( */ export const flushWAL = async (): Promise => { const c = conn; - if (!c) return; + // CHECKPOINT on a read-only connection can leave LadybugDB's checkpoint + // marker set even though the write is rejected, poisoning the next reader. + if (!c || currentDbReadOnly) return; try { await withConnLock(async () => { const checkpointResult = await c.query('CHECKPOINT'); @@ -2428,7 +2475,7 @@ export const flushWAL = async (): Promise => { */ export const tryFlushWAL = async (): Promise => { const c = conn; - if (!c) return false; + if (!c || currentDbReadOnly) return false; // Runs on the periodic WAL-checkpoint driver. The lock makes this CHECKPOINT // wait for any in-flight COPY / writeback on the singleton connection instead // of executing concurrently with it (the `analyze --pdg` heap-corruption bug). @@ -2449,7 +2496,13 @@ export const tryFlushWAL = async (): Promise => { * @see flushWAL — CHECKPOINT-only (connection stays open) * @see closeLbug — safeClose + module state reset (full teardown) */ -export const safeClose = async (): Promise => { +export const safeClose = async (options: { strict?: boolean } = {}): Promise => { + const strict = + options.strict === true || + nativeCloseUnverified || + currentDbBufferPoolBytes !== undefined || + currentDbThreads !== undefined; + if (strict) nativeCloseUnverified = true; await flushWAL(); // Capture before close — currentDbPath stays set so the Windows post-close // probe below knows which file to wait on. @@ -2458,7 +2511,8 @@ export const safeClose = async (): Promise => { try { // eslint-disable-next-line no-restricted-syntax -- sole authorised close site await conn.close(); - } catch { + } catch (error) { + if (strict) throw error; /* best-effort */ } conn = null; @@ -2467,7 +2521,8 @@ export const safeClose = async (): Promise => { try { // eslint-disable-next-line no-restricted-syntax -- sole authorised close site await db.close(); - } catch { + } catch (error) { + if (strict) throw error; /* best-effort */ } db = null; @@ -2480,6 +2535,7 @@ export const safeClose = async (): Promise => { if (process.platform === 'win32' && closingDbPath) { const released = await waitForWindowsHandleRelease(closingDbPath); if (!released) { + if (strict) throw new Error('LadybugDB native resource reservation has not been released'); // Probe exhausted with a lock code still in flight. The next // openLbugConnection will absorb whatever residual lag remains, but // a chronic warning helps operators spot AV interference (Windows @@ -2493,6 +2549,7 @@ export const safeClose = async (): Promise => { if (closingDbPath) { await finalizeLbugSidecarsAfterClose(closingDbPath, { logger }); } + nativeCloseUnverified = false; }; /** @@ -2528,15 +2585,26 @@ export const closeLbugBeforeExit = async (): Promise => { // double-free. }; -export const closeLbug = async (): Promise => { - await safeClose(); +export const closeLbug = async (options: { strict?: boolean } = {}): Promise => { + await safeClose(options); currentDbPath = null; + currentDbBufferPoolBytes = undefined; + currentDbThreads = undefined; ftsLoaded = false; vectorExtensionLoaded = false; vectorIndexEnsured = false; ensuredFTSIndexes.clear(); }; +/** A new analysis phase may reuse the prior phase's reservation only after its + * native handles are gone. Wait for session work; never close an active query. */ +export const drainAnalysisNativeResources = async (): Promise => { + await runWithSessionLock(async () => { + if (conn || db || nativeCloseUnverified) await closeLbug({ strict: true }); + await drainLbugResourceReleases(); + }); +}; + /** * Thrown by {@link wipeLbugDbFiles} when a data-bearing member of the * LadybugDB file family is still present after the bounded diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index 2175da5e1..b39e81e59 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -847,6 +847,9 @@ type LbugModule = typeof lbug; export interface LbugDatabaseOptions { readOnly?: boolean; throwOnWalReplayFailure?: boolean; + /** Explicit per-handle reservation; omitted preserves the existing policy. */ + bufferPoolBytes?: number; + threads?: number; } export interface LbugConnectionHandle { @@ -854,6 +857,61 @@ export interface LbugConnectionHandle { conn: lbug.Connection; } +interface NativeRelease { + db: lbug.Database; + conn?: lbug.Connection; + databasePath?: string; + connectionClosed: boolean; + databaseClosed: boolean; + pending?: Promise; +} + +// Only closing/failed handles are held strongly. A rejected close is not proof +// that its native pool was released, including failed connection construction. +const nativeReleases = new Map(); +const reservedHandlePaths = new WeakMap(); + +function releaseNativeResources( + handle: Pick & Partial>, + databasePath?: string, +): Promise { + let release = nativeReleases.get(handle.db); + if (!release) { + release = { ...handle, databasePath, connectionClosed: !handle.conn, databaseClosed: false }; + nativeReleases.set(handle.db, release); + } + if (release.pending) return release.pending; + const state = release; + state.pending = (async () => { + if (!state.connectionClosed) { + if (state.conn) await state.conn.close(); + state.connectionClosed = true; + } + if (!state.databaseClosed) { + await state.db.close(); + state.databaseClosed = true; + } + if ( + process.platform === 'win32' && + state.databasePath && + !(await waitForWindowsHandleRelease(state.databasePath)) + ) + throw new Error('LadybugDB native resource reservation has not been released'); + nativeReleases.delete(state.db); + })().finally(() => { + state.pending = undefined; + }); + return state.pending; +} + +/** Before admitting another analysis phase, retry only previously requested + * closes. Active readers/writers are never closed or interrupted here. */ +export async function drainLbugResourceReleases(): Promise { + for (const release of nativeReleases.values()) { + await releaseNativeResources(release, release.databasePath); + } +} + /** * Return true when the error message indicates that a LadybugDB write * transaction could not proceed due to lock contention — either a file @@ -946,10 +1004,15 @@ export function createLbugDatabase( databasePath: string, options: LbugDatabaseOptions = {}, ): lbug.Database { + if ( + options.bufferPoolBytes !== undefined && + (!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes <= 0) + ) + throw new Error('Explicit LadybugDB buffer pool must be a positive byte count'); // .d.ts declares fewer args than the native constructor accepts. return new (lbugModule.Database as any)( databasePath, - resolveBufferManagerSize(), // bufferManagerSize (#2557: default min(2 GiB, 80% RAM); GITNEXUS_LBUG_BUFFER_POOL_SIZE overrides; 0 restores the native 80%-of-RAM default) + options.bufferPoolBytes ?? resolveBufferManagerSize(), // Existing callers retain the environment/default policy. false, // enableCompression (pinned for v0.16.0) options.readOnly ?? false, LBUG_MAX_DB_SIZE, @@ -1145,18 +1208,50 @@ export async function openLbugConnection( databasePath: string, options: LbugDatabaseOptions = {}, ): Promise { + if ( + options.threads !== undefined && + (!Number.isSafeInteger(options.threads) || options.threads < 1) + ) + throw new Error('LadybugDB threads must be a positive integer'); + const reserved = options.bufferPoolBytes !== undefined || options.threads !== undefined; + if (reserved) await drainLbugResourceReleases(); const safePath = toNativeSafePath(databasePath); let db: lbug.Database | undefined; try { db = await openWithLockRetry(() => createLbugDatabase(lbugModule, safePath, options), safePath); - return { db, conn: new lbugModule.Connection(db) }; + const conn = + options.threads === undefined + ? new lbugModule.Connection(db) + : new lbugModule.Connection(db, options.threads); + const handle = { db, conn }; + if (reserved) reservedHandlePaths.set(handle, safePath); + return handle; } catch (err) { - if (db) await db.close().catch(() => {}); + if (db) { + if (reserved) { + try { + await releaseNativeResources({ db }, safePath); + } catch (releaseError) { + throw new AggregateError( + [err, releaseError], + 'LadybugDB open failed and resource release is unverified', + { cause: err }, + ); + } + } else await db.close().catch(() => {}); + } throw err; } } -export async function closeLbugConnection(handle: LbugConnectionHandle): Promise { +export async function closeLbugConnection( + handle: LbugConnectionHandle, + options: { strict?: boolean; databasePath?: string } = {}, +): Promise { + if (options.strict || reservedHandlePaths.has(handle) || nativeReleases.has(handle.db)) { + await releaseNativeResources(handle, options.databasePath ?? reservedHandlePaths.get(handle)); + return; + } await handle.conn.close().catch(() => {}); await handle.db.close().catch(() => {}); } diff --git a/gitnexus/src/core/lbug/legacy-vector-export.ts b/gitnexus/src/core/lbug/legacy-vector-export.ts new file mode 100644 index 000000000..8f6744d32 --- /dev/null +++ b/gitnexus/src/core/lbug/legacy-vector-export.ts @@ -0,0 +1,185 @@ +/** Disk-backed migration of the legacy vector table, without copying its graph. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import lbug from '@ladybugdb/core'; +import { openLbugConnection, closeLbugConnection, toNativeSafePath } from './lbug-config.js'; +import { escapeCypherString } from './cypher-escape.js'; +import { assertLbugPublicationReady } from './publication.js'; +import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js'; +import { + recordLegacyEmbeddingSource, + type LegacyEmbeddingSourceRecord, +} from '../../storage/legacy-embedding-source.js'; + +export interface LegacyVectorExportOptions { + metadataDirectory: string; + /** The caller owns the slot lock and a stable source connection. Only catalog, + * count and native COPY results cross this callback, never vector payloads. */ + query: (cypher: string) => Promise; + provenance: Pick< + LegacyEmbeddingSourceRecord, + 'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity' + >; + excluded: Iterable | AsyncIterable; + /** Additional native pool, admitted alongside the source's pool. */ + bufferPoolBytes: number; + threads: number; + signal?: AbortSignal; +} + +const fields = ['id', 'nodeId', 'chunkIndex', 'startLine', 'endLine', 'embedding', 'contentHash']; +const countQuery = 'MATCH (e:CodeEmbedding) RETURN count(e) AS count'; + +/** Preserve the old physical width before writable schema initialization can + * replace its table. The caller publishes the returned receipt under its lock. */ +export async function exportLegacyVectorsFromFile( + sourcePath: string, + options: Omit & { sourcePoolBytes: number }, +): Promise { + options.signal?.throwIfAborted(); + const stat = await fs.lstat(sourcePath); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy vector source must be an owned regular file'); + const handle = await openLbugConnection(lbug, sourcePath, { + readOnly: true, + bufferPoolBytes: options.sourcePoolBytes, + threads: options.threads, + }); + try { + return await exportLegacyVectors({ + ...options, + query: async (sql) => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + if (cursors.length !== 1) throw new Error('Unexpected legacy vector export result'); + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }, + }); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: sourcePath }); + } +} + +function count(rows: unknown[]): number { + const value = (rows[0] as { count?: unknown } | undefined)?.count; + if (rows.length !== 1 || (typeof value !== 'number' && typeof value !== 'bigint')) + throw new Error('Legacy vector export count is unverified'); + const n = Number(value); + if (!Number.isSafeInteger(n) || n < 0) throw new Error('Legacy vector export count is invalid'); + return n; +} + +/** The returned receipt pins an independent, immutable table-only database. + * Group completeness/input compatibility is still checked by LegacyVectorSource; + * copying a row does not certify its historical provider or text configuration. + * The source and its metadata are never changed here. Publish the returned + * reference under the slot lock before any source mutation or cleanup. */ +export async function exportLegacyVectors( + options: LegacyVectorExportOptions, +): Promise { + options.signal?.throwIfAborted(); + if ( + !Number.isSafeInteger(options.bufferPoolBytes) || + options.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(options.threads) || + options.threads < 1 + ) + throw new Error('Legacy vector export requires an admitted native pool'); + const root = await embeddingJobDirectory(options.metadataDirectory); + const expected = count(await options.query(countQuery)); + if (expected === 0) return undefined; + const catalog = (await options.query("CALL table_info('CodeEmbedding') RETURN *")) as { + name?: unknown; + type?: unknown; + 'primary key'?: unknown; + }[]; + const vectorType = catalog.find((column) => column.name === 'embedding')?.type; + const match = typeof vectorType === 'string' && /^FLOAT\[([1-9]\d*)\]$/.exec(vectorType); + const dimensions = match ? Number(match[1]) : NaN; + const types = ['STRING', 'STRING', 'INT32', 'INT64', 'INT64', vectorType, 'STRING']; + if ( + !Number.isSafeInteger(dimensions) || + dimensions < 1 || + fields.some((name, i) => !catalog.some((c) => c.name === name && c.type === types[i])) || + !catalog.some((c) => c.name === 'id' && c['primary key'] === true) || + (options.provenance.recordedIdentity?.dimensions !== undefined && + options.provenance.recordedIdentity.dimensions !== dimensions) + ) + throw new Error('Legacy vector schema differs from its recorded provenance; source retained'); + + const id = randomUUID(); + const staging = path.join(root, `lbug.staging.${id}`); + const csv = staging + '.csv'; + const sourceFile = `lbug.generation.${id.replaceAll('-', '')}`; + const destination = path.join(root, sourceFile); + const csvLiteral = `'${escapeCypherString(toNativeSafePath(csv).replace(/\\/g, '/'))}'`; + let stagingReleased = true; + try { + options.signal?.throwIfAborted(); + await options.query( + `COPY (MATCH (e:CodeEmbedding) RETURN ${fields.map((f) => `e.${f} AS ${f}`).join(', ')}) TO ${csvLiteral} (HEADER=true)`, + ); + options.signal?.throwIfAborted(); + stagingReleased = false; + const handle = await openLbugConnection(lbug, staging, { + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }); + const query = async (sql: string): Promise => { + const result = await handle.conn.query(sql); + const results = Array.isArray(result) ? result : [result]; + try { + if (results.length !== 1) throw new Error('Unexpected legacy vector export result'); + return await results[0].getAll(); + } finally { + for (const cursor of results) cursor.close(); + } + }; + try { + await query( + `CREATE NODE TABLE CodeEmbedding (${fields.map((name, i) => `${name} ${types[i]}`).join(', ')}, PRIMARY KEY(id))`, + ); + await query(`COPY CodeEmbedding FROM ${csvLiteral} (HEADER=true, auto_detect=false)`); + if ( + count(await query(countQuery)) !== expected || + count(await options.query(countQuery)) !== expected + ) + throw new Error('Legacy vector export lost rows or its source changed; source retained'); + options.signal?.throwIfAborted(); + await query('CHECKPOINT'); + } finally { + // A failed physical release remains charged by the native reservation + // manager. Never rename or certify that still-open export. + await closeLbugConnection(handle, { strict: true, databasePath: staging }); + stagingReleased = true; + } + options.signal?.throwIfAborted(); + await assertLbugPublicationReady(staging); + await fs.rename(staging, destination); + const file = await fs.open(destination, 'r+'); + try { + await file.sync(); + } finally { + await file.close(); + } + await syncEmbeddingJobDirectory(root); + return await recordLegacyEmbeddingSource( + root, + { sourceFile, ...options.provenance }, + options.excluded, + options.signal, + ); + } finally { + await fs.rm(csv, { force: true }); + // The original source remains untouched. A failed close must retain its + // live file; renamed generations are reclaimed by the ownership sweep, + // since a receipt write can succeed before reporting a durability failure. + if (stagingReleased) + for (const suffix of ['', '.wal', '.shadow']) await fs.rm(staging + suffix, { force: true }); + } +} diff --git a/gitnexus/src/core/lbug/legacy-vector-source.ts b/gitnexus/src/core/lbug/legacy-vector-source.ts new file mode 100644 index 000000000..eccb59f46 --- /dev/null +++ b/gitnexus/src/core/lbug/legacy-vector-source.ts @@ -0,0 +1,388 @@ +/** Bounded, read-only reuse of verified legacy monolithic embedding groups. */ +import lbug, { type LbugValue } from '@ladybugdb/core'; +import fs from 'node:fs/promises'; +import { + openLbugConnection, + closeLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; +import { + VectorStore, + type LegacyVectorGroup, + type VectorNodeReceipt, + type StoredVectorChunk, +} from './vector-store.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import type { EmbeddingIdentity } from '../embeddings/embedding-identity.js'; +import { resolveEmbeddingConfig } from '../embeddings/config.js'; +import { + iterateEmbeddingInputs, + type EmbeddingReuseOptions, +} from '../embeddings/embedding-inputs.js'; +import { isShortLabel, type EmbeddableNode, type EmbeddingConfig } from '../embeddings/types.js'; + +export interface LegacyVectorSourceOptions { + /** Caller validates ownership and holds the slot lock. WAL recovery and + * checkpointing must already have completed in the graph process. */ + path: string; + sourceIdentity: string; + recordedIdentity?: Partial; + expectedIdentity: EmbeddingIdentity; + /** Optional historical input receipt; never synthesized from current config. */ + recordedInputIdentity?: string; + expectedInputIdentity: string; + config?: Partial; + bufferPoolBytes: number; + threads: number; + batchBytes: number; + signal?: AbortSignal; + log?: (message: string) => void; +} + +/** A row hash does not identify its provider. Never fill missing historical + * fields from today's configuration, including for a mere file relocation. */ +export function legacyVectorCompatibility( + recorded: Partial | undefined, + expected: EmbeddingIdentity, +): string | undefined { + if ( + !recorded || + typeof recorded.provider !== 'string' || + !/^(local|http:[a-f0-9]{64})$/.test(recorded.provider) || + typeof recorded.model !== 'string' || + !recorded.model || + !Number.isSafeInteger(recorded.dimensions) || + !recorded.dimensions || + recorded.dimensions < 1 + ) + return 'Legacy embedding provider/model/dimensions provenance is unknown; source retained.'; + if ( + recorded.provider !== expected.provider || + recorded.model !== expected.model || + recorded.dimensions !== expected.dimensions + ) + return 'Legacy embedding provider/model/dimensions differ from this request; source retained.'; + return undefined; +} + +const count = (value: unknown): number => { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid legacy vector count'); + const n = Number(value); + if (!Number.isSafeInteger(n) || n < 0) throw new Error('Invalid legacy vector count'); + return n; +}; +type Row = Record; +function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult { + if (!Array.isArray(result)) return result; + if (result.length === 1) return result[0]; + for (const part of result) part.close(); + throw new Error('Unexpected legacy query result count'); +} + +export class LegacyVectorSource { + readonly requiresInputValidation = true; + private readonly statements = new Map(); + private readonly config: EmbeddingConfig; + private active = false; + private closed = false; + private closing?: Promise; + private constructor( + readonly options: LegacyVectorSourceOptions, + private readonly handle: LbugConnectionHandle, + private readonly index: VectorStore, + ) { + this.config = resolveEmbeddingConfig(options.config); + } + + static async open( + options: LegacyVectorSourceOptions, + index: VectorStore, + excluded: () => AsyncIterable | Iterable, + ): Promise { + const incompatible = legacyVectorCompatibility( + options.recordedIdentity, + options.expectedIdentity, + ); + if (incompatible) throw new Error(incompatible); + if ( + !/^[a-f0-9]{64}$/.test(options.sourceIdentity) || + !/^[a-f0-9]{64}$/.test(options.expectedInputIdentity) || + options.expectedIdentity.dimensions !== index.options.dimensions || + !Number.isSafeInteger(options.batchBytes) || + options.batchBytes < 1024 + ) + throw new Error('Invalid legacy vector reuse reservation'); + if ( + options.recordedInputIdentity !== undefined && + options.recordedInputIdentity !== options.expectedInputIdentity + ) + throw new Error( + 'Legacy embedding input provenance differs from this request; source retained.', + ); + if (!options.recordedInputIdentity) + options.log?.( + 'Legacy multi-chunk input provenance is unknown (chunk size/overlap); source retained and those groups require regeneration.', + ); + options.signal?.throwIfAborted(); + const stat = await fs.lstat(options.path); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy vectors require an owned regular database'); + const source = new LegacyVectorSource( + options, + await openLbugConnection(lbug, options.path, { + readOnly: true, + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }), + index, + ); + try { + await index.prepareLegacyReuseIndex( + options.sourceIdentity, + () => source.groups(), + excluded, + options.signal, + ); + return source; + } catch (error) { + await source.close(); + throw error; + } + } + + private acquire(): void { + if (this.active || this.closed) throw new Error('Legacy vector source is closed or active'); + this.options.signal?.throwIfAborted(); + this.active = true; + } + private async result(sql: string, params?: Record): Promise { + if (!params) return single(await this.handle.conn.query(sql)); + let prepared = this.statements.get(sql); + if (!prepared) { + prepared = await this.handle.conn.prepare(sql); + if (!prepared.isSuccess()) throw new Error(prepared.getErrorMessage()); + this.statements.set(sql, prepared); + } + return single(await this.handle.conn.execute(prepared, params)); + } + + private async *groups(): AsyncGenerator { + this.acquire(); + let result: lbug.QueryResult | undefined; + try { + // Native sorting is charged to this explicit pool; JS retains one group + // and never fetches vector payloads while building the receipt index. + result = await this.result( + 'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash, e.startLine AS startLine, e.endLine AS endLine ORDER BY e.nodeId, e.chunkIndex', + ); + let group: LegacyVectorGroup | undefined; + let valid = true; + while (result.hasNext()) { + this.options.signal?.throwIfAborted(); + const row = result.getNextSync(); + if (typeof row.nodeId !== 'string') throw new Error('Invalid legacy node identity'); + assertValidGraphNodeId(row.nodeId, 'Legacy vector source'); + if (group?.nodeId !== row.nodeId) { + if (group && valid) yield group; + group = { + nodeId: row.nodeId, + contentHash: typeof row.contentHash === 'string' ? row.contentHash : '', + chunks: 0, + sourceStartLine: count(row.startLine), + sourceEndLine: count(row.endLine), + }; + valid = Boolean(group.contentHash); + } + if (!group) throw new Error('Missing legacy group'); + const ordinal = count(row.chunkIndex), + start = count(row.startLine), + end = count(row.endLine); + if ( + ordinal !== group.chunks || + row.id !== `${group.nodeId}:${ordinal}` || + row.contentHash !== group.contentHash || + end < start + ) + valid = false; + group.chunks++; + group.sourceEndLine = Math.max(group.sourceEndLine, end); + } + if (group && valid) yield group; + } finally { + try { + result?.close(); + } finally { + this.active = false; + } + } + } + + private async rows( + receipt: VectorNodeReceipt, + start: number, + length: number, + vectors: boolean, + ): Promise { + const result = await this.result( + `UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.contentHash AS contentHash${vectors ? ', e.embedding AS embedding' : ''}`, + { + ids: Array.from({ length }, (_, i) => `${receipt.nodeId}:${start + i}`), + }, + ); + const rows: Row[] = []; + try { + while (result.hasNext()) { + if (rows.length >= length) throw new Error('Legacy query exceeded its bounded result size'); + rows.push(result.getNextSync()); + } + } finally { + result.close(); + } + rows.sort((a, b) => count(a.chunkIndex) - count(b.chunkIndex)); + if ( + rows.length !== length || + rows.some( + (row, i) => + row.nodeId !== receipt.nodeId || + row.contentHash !== receipt.contentHash || + count(row.chunkIndex) !== start + i, + ) + ) + throw new Error('Legacy complete group changed; source retained'); + return rows; + } + + /** Legacy hashes did not record chunk-size/overlap. Multiple chunks may have + * identical line spans but different character boundaries, so their layout + * cannot replace an input receipt. Short labels ignore those settings; a + * complete single-chunk group covers its full input without overlap. */ + async reusable( + nodeId: string, + contentHash: string, + options: EmbeddingReuseOptions, + ): Promise { + const { node, prepareInputValidation, inputs, matchesHash } = options; + this.acquire(); + const originalMethods = node.methodNames, + originalFields = node.fieldNames; + try { + const group = await this.index.legacyReuseGroup(this.options.sourceIdentity, nodeId); + if (!group || node.id !== nodeId) return null; + const sourceAware = /^[a-f0-9]{40}:[a-f0-9]{40}$/.test(group.contentHash); + if (group.contentHash !== contentHash && !(await matchesHash?.(group.contentHash))) + return null; + if ( + !this.options.recordedInputIdentity && + !sourceAware && + !isShortLabel(node.label) && + group.chunks > 1 + ) { + return null; + } + await prepareInputValidation?.(); + const receipt: VectorNodeReceipt = { ...group, complete: true }; + const rowBytes = 512 + 8 * nodeId.length; + if (rowBytes > this.options.batchBytes) + throw new Error('Legacy metadata does not fit its reservation'); + const blockSize = Math.min( + 64, + Math.max(1, Math.floor(Math.min(32768, this.options.batchBytes) / rowBytes)), + ); + let block: Row[] = [], + ordinal = 0, + delta: number | undefined; + for await (const span of inputs ? inputs() : this.inputSpans(node)) { + if (ordinal >= group.chunks) return null; + if (ordinal % blockSize === 0) + block = await this.rows( + receipt, + ordinal, + Math.min(blockSize, group.chunks - ordinal), + false, + ); + const row = block[ordinal % blockSize]; + const start = count(row.startLine), + end = count(row.endLine); + delta ??= span.startLine - start; + if (start + delta !== span.startLine || end + delta !== span.endLine) return null; + ordinal++; + } + if (ordinal !== group.chunks || delta === undefined || (node.startLine ?? 0) - delta < 0) + return null; + // Legacy rows recorded chunk ranges, not the original node's end. + // Do not let the copy path clamp the final chunk to an invented range. + return { + ...receipt, + sourceStartLine: (node.startLine ?? 0) - delta, + sourceEndLine: undefined, + }; + } finally { + node.methodNames = originalMethods; + node.fieldNames = originalFields; + this.active = false; + } + } + + private async *inputSpans( + node: EmbeddableNode, + ): AsyncGenerator<{ startLine: number; endLine: number }> { + yield* iterateEmbeddingInputs(node, this.config, { signal: this.options.signal }); + } + + async readChunks( + receipt: VectorNodeReceipt, + start: number, + length: number, + ): Promise { + if ( + !Number.isSafeInteger(start) || + start < 0 || + !Number.isSafeInteger(length) || + length < 1 || + length > 256 || + start + length > receipt.chunks || + length * (32 * this.options.expectedIdentity.dimensions + 1024 + 4 * receipt.nodeId.length) > + this.options.batchBytes + ) + throw new Error('Invalid legacy vector read batch'); + this.acquire(); + try { + return (await this.rows(receipt, start, length, true)).map((row) => { + if ( + !Array.isArray(row.embedding) || + row.embedding.length !== this.options.expectedIdentity.dimensions || + row.embedding.some((value) => typeof value !== 'number' || !Number.isFinite(value)) + ) + throw new Error('Invalid legacy vector dimensions or values'); + const startLine = count(row.startLine), + endLine = count(row.endLine); + if (endLine < startLine) throw new Error('Invalid legacy vector lines'); + return { + chunkIndex: count(row.chunkIndex), + startLine, + endLine, + embedding: row.embedding as number[], + }; + }); + } finally { + this.active = false; + } + } + async close(): Promise { + if (this.closing) return this.closing; + if (this.active) throw new Error('Cannot close an active legacy vector read'); + this.closed = true; + this.statements.clear(); + this.closing = closeLbugConnection(this.handle, { + strict: true, + databasePath: this.options.path, + }); + try { + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; + } + } +} diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index 458c29918..ab64ae9b1 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -60,6 +60,9 @@ interface PoolEntry { }>; lastUsed: number; dbPath: string; + /** Immutable vector hard links share one native pool, independently of the + * path each checkout retains. Graph and injected handles remain path-keyed. */ + dbCacheKey?: string; /** Filesystem identity of the on-disk DB at open time. When `analyze` * rebuilds or mutates the index, this diverges from the current file and * initLbug re-opens the pool onto the new file instead of serving the @@ -68,6 +71,14 @@ interface PoolEntry { dbIdentity: DbIdentity | null; /** Set to true when the pool entry is closed — checkin will close orphaned connections */ closed: boolean; + artifactOptions?: VectorPoolOptions; +} + +/** Separate vector files never inherit the graph's default native pool. */ +export interface VectorPoolOptions { + kind: 'vector'; + bufferPoolBytes: number; + threads: number; } /** Filesystem identity used to detect an index rebuilt/mutated under a live @@ -131,9 +142,8 @@ export function addPoolCloseListener(listener: PoolCloseListener): () => void { } /** - * Shared Database cache keyed by resolved dbPath. - * Multiple repoIds pointing to the same path share one native Database - * object to avoid exhausting the buffer manager's mmap budget. + * Shared Database cache keyed by graph path or immutable vector file identity. + * Sibling hard links must not multiply a sealed artifact's native reservation. */ interface SharedDB { db: lbug.Database; @@ -157,6 +167,7 @@ interface SharedDB { dbIdentity?: DbIdentity | null; /** When true, closeOne skips db.close() — the Database is owned externally. */ external?: boolean; + artifactOptions?: VectorPoolOptions; } const dbCache = new Map(); @@ -348,12 +359,17 @@ export const getMaxResidentRepos = (): number => MAX_POOL_SIZE; * entry is pinned, no eviction occurs and the pool transiently exceeds * MAX_POOL_SIZE (see the pinnedRepos docstring). */ -async function evictLRU(): Promise { - if (pool.size < MAX_POOL_SIZE) return; +async function evictLRU(kind?: 'vector'): Promise { + // A vector artifact must not halve the existing graph-cache capacity. Each + // lane has its own bounded allowance; vector handles reserve only 64 MiB. + let residents = 0; + for (const entry of pool.values()) if (entry.artifactOptions?.kind === kind) residents++; + if (residents < MAX_POOL_SIZE) return; let oldestId: string | null = null; let oldestTime = Infinity; for (const [id, entry] of pool) { + if (entry.artifactOptions?.kind !== kind) continue; if (pinnedRepos.has(id)) continue; if (entry.checkedOut === 0 && entry.lastUsed < oldestTime) { oldestTime = entry.lastUsed; @@ -431,7 +447,8 @@ async function closeOne(repoId: string): Promise { // Only close the Database when no other repoIds reference it. // External databases (injected via initLbugWithDb) are never closed here — // the core adapter owns them and handles their lifecycle. - const shared = dbCache.get(entry.dbPath); + const cacheKey = entry.dbCacheKey ?? entry.dbPath; + const shared = dbCache.get(cacheKey); if (shared) { shared.refCount--; if (shared.refCount === 0) { @@ -449,7 +466,7 @@ async function closeOne(repoId: string): Promise { // subsequent reopen (evictLRU / the "idle & changed" path below) must not // race. See the awaited call site in evictLRU for the full rationale. await shared.db.close().catch(() => {}); - dbCache.delete(entry.dbPath); + dbCache.delete(cacheKey); } } } @@ -783,10 +800,24 @@ function ladybugUnavailableError(repoId: string, err: Error | undefined): Error * bookkeeping on "did the pool actually roll over" (LocalBackend) use the * return value; callers that only need the pool ready can ignore it. */ -export const initLbug = async (repoId: string, dbPath: string): Promise => { +export const initLbug = async ( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise => { + if ( + artifactOptions && + (artifactOptions.kind !== 'vector' || + !Number.isSafeInteger(artifactOptions.bufferPoolBytes) || + artifactOptions.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(artifactOptions.threads) || + artifactOptions.threads < 1) + ) + throw new Error('Invalid vector reader resource reservation'); + const reservation = artifactOptions ? Object.freeze({ ...artifactOptions }) : undefined; let lastError: Error | undefined; for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { - const result = await withPoolLock(() => initLbugInner(repoId, dbPath)); + const result = await withPoolLock(() => initLbugInner(repoId, dbPath, reservation)); if (result.status === 'done') return result.reopened; lastError = result.error; if (attempt === LOCK_RETRY_ATTEMPTS) break; @@ -795,7 +826,11 @@ export const initLbug = async (repoId: string, dbPath: string): Promise throw ladybugUnavailableError(repoId, lastError); }; -const initLbugInner = async (repoId: string, dbPath: string): Promise => { +const initLbugInner = async ( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise => { const existing = pool.get(repoId); if (existing) { existing.lastUsed = Date.now(); @@ -804,7 +839,12 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise 0) return { status: 'done', reopened: false }; - // Closing only this consumer would leave the shared native handle alive + // For graph paths, closing only this consumer leaves the shared handle alive // and doInitLbug would attach eight new connections to that same old // generation. Keep this pool intact until the other consumers release it; // its opened identity stays divergent so the next init can retry then. - const shared = dbCache.get(existing.dbPath); - if (shared && shared.db === existing.db && shared.refCount > 1) { + const shared = dbCache.get(existing.dbCacheKey ?? existing.dbPath); + if ( + !artifactOptions && + existing.dbPath === dbPath && + shared && + shared.db === existing.db && + shared.refCount > 1 + ) { return { status: 'done', reopened: false }; } // Awaited: see the rationale on the evictLRU call site in doInitLbug below. await closeOne(repoId); // idle & changed → evict, then fall through to reopen the new file } - return doInitLbug(repoId, dbPath); + return doInitLbug(repoId, dbPath, artifactOptions); }; +function sameArtifactOptions(a?: VectorPoolOptions, b?: VectorPoolOptions): boolean { + return ( + a?.kind === b?.kind && a?.bufferPoolBytes === b?.bufferPoolBytes && a?.threads === b?.threads + ); +} + +/** Validate every retained path before sharing a native handle. No content + * hash or copy: sealed files can be pinned by hard links without extra pools. + * An unavailable inode identity falls back to the existing path isolation. */ +async function vectorDatabaseKey(dbPath: string): Promise { + const stat = await fs.lstat(dbPath, { bigint: true }); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector artifact must be a regular file'); + for (const suffix of ['.wal', '.shadow']) { + try { + const sidecar = await fs.lstat(dbPath + suffix); + if (sidecar.size || !sidecar.isFile() || sidecar.isSymbolicLink()) + throw new Error('Published vector artifact has unfinished native writes'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + // NUL cannot occur in a filesystem path, so these keys cannot collide with + // the graph cache. ctime changes when another hard link is added; mtime does not. + return typeof stat.ino === 'bigint' && + stat.ino > 0n && + typeof stat.dev === 'bigint' && + stat.dev > 0n + ? `vector\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}` + : `vector\0path\0${dbPath}`; +} + +/** Called after vectorDatabaseKey validates the immutable file and sidecars. + * Readers never quarantine, replay, or repair their writer's sidecars. */ +async function openVectorReadOnlyDatabase( + dbPath: string, + options: VectorPoolOptions, +): Promise { + silenceStdout(); + let db: lbug.Database | undefined; + try { + db = createLbugDatabase(lbug, toNativeSafePath(dbPath), { + readOnly: true, + bufferPoolBytes: options.bufferPoolBytes, + }); + await db.init(); + return db; + } catch (error) { + if (db) await db.close().catch(() => {}); + throw error; + } finally { + restoreStdout(); + } +} + /** * Internal init — creates DB, pre-warms connections, loads FTS, then registers pool. * Pool entry is registered LAST so concurrent executeQuery calls see either * "not initialized" (and throw) or a fully ready pool — never a half-built one. */ -async function doInitLbug(repoId: string, dbPath: string): Promise { +async function doInitLbug( + repoId: string, + dbPath: string, + artifactOptions?: VectorPoolOptions, +): Promise { // Check if database exists try { await fs.stat(dbPath); @@ -854,11 +959,14 @@ async function doInitLbug(repoId: string, dbPath: string): Promise connection.close())); + shared.refCount--; + if (shared.refCount === 0) { + dbCache.delete(cacheKey); + if (!shared.external) await db.close().catch(() => {}); + } + throw error; } finally { preWarmActive = false; } @@ -955,7 +1082,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { throw new Error(`LadybugDB not initialized for repo "${repoId}". Call initLbug first.`); } - const shared = dbCache.get(entry.dbPath); + const shared = dbCache.get(entry.dbCacheKey ?? entry.dbPath); if (!shared) { throw new Error(`LadybugDB shared handle is unavailable for repo "${repoId}".`); } diff --git a/gitnexus/src/core/lbug/vector-store.ts b/gitnexus/src/core/lbug/vector-store.ts new file mode 100644 index 000000000..85a9f9cfa --- /dev/null +++ b/gitnexus/src/core/lbug/vector-store.ts @@ -0,0 +1,1026 @@ +/** An exclusively owned vector artifact; never opens the graph singleton. */ +import lbug, { type LbugValue } from '@ladybugdb/core'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + closeLbugConnection, + openLbugConnection, + type LbugConnectionHandle, +} from './lbug-config.js'; +import { ExtensionManager } from './extension-loader.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import { embeddingJobDirectory } from '../../storage/embedding-job.js'; +import { getExactScanLimit } from '../platform/capabilities.js'; +import type { EmbeddingReuseOptions } from '../embeddings/embedding-inputs.js'; +import { logger } from '../logger.js'; + +export const VECTOR_STORE_SCHEMA = 'gitnexus-vector-store-v2'; +const BLOCK_ROWS = 256; +const MAX_CHUNKS = 2147483647; + +export interface StoredVectorChunk { + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; +} +export interface VectorNodeReceipt { + nodeId: string; + contentHash: string; + chunks: number; + complete: boolean; + sourceStartLine?: number; + sourceEndLine?: number; +} +export interface VectorNodeInput { + nodeId: string; + contentHash: string; + sourceStartLine?: number; + sourceEndLine?: number; +} +/** A complete legacy group, outside every interrupted checkpoint window. + * This is a lookup proof only; it is never counted as current paid coverage. */ +export interface LegacyVectorGroup { + nodeId: string; + contentHash: string; + chunks: number; + sourceStartLine: number; + sourceEndLine: number; +} +export type VectorWriteChunk = StoredVectorChunk & VectorNodeInput; +export interface VectorStoreOptions { + directory: string; + identity: string; + dimensions: number; + bufferPoolBytes: number; + threads: number; + mode: 'create' | 'resume' | 'read'; + /** Only before a scheduling journal exists: an interrupted initial DDL + * transaction may leave an owned database with a verified empty catalog. */ + recoverEmptyBootstrap?: boolean; + /** Maximum JS vector batch retained by the caller and passed into native. */ + batchBytes: number; +} + +type Row = Record; +const nonnegative = (n: unknown): n is number => + typeof n === 'number' && Number.isSafeInteger(n) && n >= 0; +const count = (value: unknown): number => { + if (typeof value !== 'number' && typeof value !== 'bigint') + throw new Error('Invalid vector-store count'); + const n = Number(value); + if (!nonnegative(n)) throw new Error('Invalid vector-store count'); + return n; +}; + +export class VectorStore { + private readonly prepared = new Map(); + private busy = false; + private closed = false; + private closing?: Promise; + private poisoned = false; + private sealed = false; + private selectionReady = false; + + get isSealed(): boolean { + return this.sealed; + } + + private constructor( + private readonly handle: LbugConnectionHandle, + readonly options: Readonly, + ) {} + + static async open(options: VectorStoreOptions): Promise { + if ( + !/^[a-f0-9]{64}$/.test(options.identity) || + !Number.isSafeInteger(options.dimensions) || + options.dimensions < 1 || + !Number.isSafeInteger(options.batchBytes) || + options.batchBytes < options.dimensions * 16 || + !Number.isSafeInteger(options.bufferPoolBytes) || + options.bufferPoolBytes < 8 * 1024 * 1024 || + !Number.isSafeInteger(options.threads) || + options.threads < 1 || + !['create', 'resume', 'read'].includes(options.mode) + ) + throw new Error('Invalid vector-store resource reservation or identity'); + const directory = await embeddingJobDirectory(options.directory); + const file = path.join(directory, 'vectors'); + for (const suffix of ['', '.wal', '.shadow', '.lock']) { + try { + const stat = await fs.lstat(file + suffix); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector artifact must be an owned regular file'); + if (suffix === '' && options.mode === 'create') + throw new Error('Vector artifact already exists'); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' || + (suffix === '' && options.mode !== 'create') + ) + throw error; + } + } + const handle = await openLbugConnection(lbug, file, { + readOnly: options.mode === 'read', + bufferPoolBytes: options.bufferPoolBytes, + threads: options.threads, + }); + const store = new VectorStore(handle, Object.freeze({ ...options, directory })); + try { + const emptyBootstrap = + options.mode === 'resume' && + options.recoverEmptyBootstrap === true && + (await store.query('CALL show_tables() RETURN *')).length === 0; + if (options.mode === 'create' || emptyBootstrap) { + // DDL is transactional in LadybugDB. A crash must expose either the + // complete schema/identity or an empty catalog, never half a bootstrap. + await store.transaction(async () => { + await store.query( + `CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[${options.dimensions}], contentHash STRING, PRIMARY KEY(id))`, + ); + await store.query( + 'CREATE NODE TABLE EmbeddingNodeReceipt (nodeId STRING, contentHash STRING, chunks INT64, complete BOOL, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', + ); + await store.query( + 'CREATE NODE TABLE EmbeddingArtifact (id STRING, identity STRING, schema STRING, dimensions INT64, complete BOOL, indexKind STRING, PRIMARY KEY(id))', + ); + await store.query( + "CREATE (:EmbeddingArtifact {id:'artifact', identity:$identity, schema:$schema, dimensions:$dimensions, complete:false, indexKind:'pending'})", + { + identity: options.identity, + schema: VECTOR_STORE_SCHEMA, + dimensions: options.dimensions, + }, + ); + }); + await store.query('CHECKPOINT'); + } else { + const rows = await store.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.identity AS identity, a.schema AS schema, a.dimensions AS dimensions, a.complete AS complete", + ); + if ( + rows.length !== 1 || + rows[0].identity !== options.identity || + rows[0].schema !== VECTOR_STORE_SCHEMA || + count(rows[0].dimensions) !== options.dimensions + ) + throw new Error('Vector artifact identity mismatch; artifacts retained'); + store.sealed = rows[0].complete === true; + if (options.mode === 'read' && !store.sealed) + throw new Error('Vector artifact is not complete'); + if (options.mode === 'resume' && !store.sealed) { + const indexes = await store.query('CALL SHOW_INDEXES() RETURN *'); + if (indexes.some((index) => index.index_name === 'code_embedding_idx')) { + const manager = new ExtensionManager(); + if ( + !(await manager.ensure((sql) => store.query(sql), 'vector', 'vector resume', { + policy: 'load-only', + })) + ) + throw new Error( + 'Cannot load VECTOR for the retained index; artifacts preserved. Restore the extension before retrying.', + ); + } + } + } + return store; + } catch (error) { + await store.close().catch(() => {}); + throw error; + } + } + + private async exclusive(action: () => Promise, write = false): Promise { + if (this.closed || this.poisoned) throw new Error('Vector store must be reopened'); + if (this.busy) throw new Error('Concurrent vector-store access is not allowed'); + if (write && (this.options.mode === 'read' || this.sealed)) + throw new Error('Published vector artifact is immutable'); + this.busy = true; + try { + return await action(); + } finally { + this.busy = false; + } + } + + private async query(sql: string, parameters?: Record): Promise { + let result: lbug.QueryResult | lbug.QueryResult[]; + if (parameters) { + let statement = this.prepared.get(sql); + if (!statement) { + statement = await this.handle.conn.prepare(sql); + if (!statement.isSuccess()) throw new Error(statement.getErrorMessage()); + this.prepared.set(sql, statement); + } + result = await this.handle.conn.execute(statement, parameters); + } else result = await this.handle.conn.query(sql); + const results = Array.isArray(result) ? result : [result]; + const rows: Row[] = []; + try { + for (const part of results) + while (part.hasNext()) { + if (rows.length >= BLOCK_ROWS) + throw new Error('Vector query exceeded its bounded result size'); + rows.push(part.getNextSync()); + } + return rows; + } finally { + for (const part of results) part.close(); + } + } + + private async transaction(action: () => Promise): Promise { + await this.query('BEGIN TRANSACTION'); + try { + await action(); + await this.query('COMMIT'); + } catch (error) { + await this.query('ROLLBACK').catch(() => { + this.poisoned = true; + }); + throw error; + } + } + + private async receipt(nodeId: string): Promise { + return (await this.receipts([nodeId])).get(nodeId) ?? null; + } + + private async receipts(ids: readonly string[]): Promise> { + const wanted = new Set(ids); + const rows = await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) RETURN n.nodeId AS nodeId, n.contentHash AS contentHash, n.chunks AS chunks, n.complete AS complete, n.sourceStartLine AS sourceStartLine, n.sourceEndLine AS sourceEndLine', + { ids: [...ids] }, + ); + const receipts = new Map(); + for (const row of rows) { + const chunks = count(row.chunks); + if ( + typeof row.nodeId !== 'string' || + !wanted.has(row.nodeId) || + receipts.has(row.nodeId) || + typeof row.contentHash !== 'string' || + !row.contentHash || + chunks > MAX_CHUNKS || + typeof row.complete !== 'boolean' + ) + throw new Error('Corrupt vector node receipt'); + receipts.set(row.nodeId, { + nodeId: row.nodeId, + contentHash: row.contentHash, + chunks, + complete: row.complete, + sourceStartLine: count(row.sourceStartLine), + sourceEndLine: count(row.sourceEndLine), + }); + } + return receipts; + } + + /** Reuse a complete group in the pending writer after validating its CURRENT + * input. Relocation updates only line metadata, with no vector round trip. + * One transaction makes an interrupted relocation safe to retry. + * resolveHash permits a caller-verified exact-input match after the source + * snapshot changed. Verification and hash/range replacement are atomic. */ + async reuseNode( + input: VectorNodeInput, + resolveHash?: (stored: string) => Promise, + ): Promise { + assertValidGraphNodeId(input.nodeId, 'Vector store'); + const startLine = input.sourceStartLine ?? 0; + const endLine = input.sourceEndLine ?? 0; + if ( + !input.contentHash || + !nonnegative(startLine) || + !nonnegative(endLine) || + endLine < startLine + ) + throw new Error('Invalid reusable embedding input'); + return this.exclusive(async () => { + const previous = await this.receipt(input.nodeId); + if (!previous?.complete || previous.chunks < 1) return null; + const contentHash = + previous.contentHash === input.contentHash + ? input.contentHash + : await resolveHash?.(previous.contentHash); + if (!contentHash) return null; + await this.verify(previous); + if ( + previous.sourceStartLine === startLine && + previous.sourceEndLine === endLine && + previous.contentHash === contentHash + ) + return previous; + const delta = startLine - (previous.sourceStartLine ?? 0); + await this.transaction(async () => { + for (let start = 0; start < previous.chunks; start += BLOCK_ROWS) { + const ids = Array.from( + { length: Math.min(BLOCK_ROWS, previous.chunks - start) }, + (_, i) => `${input.nodeId}:${start + i}`, + ); + await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) SET e.contentHash=$contentHash, e.startLine=e.startLine+$delta, e.endLine=CASE WHEN e.endLine=$oldEnd THEN $endLine ELSE e.endLine+$delta END', + { + ids, + delta, + oldEnd: previous.sourceEndLine ?? 0, + endLine, + contentHash, + }, + ); + const invalid = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) WHERE e.startLine < 0 OR e.endLine < e.startLine RETURN count(e) AS invalid', + { ids }, + ); + if (count(invalid[0].invalid)) throw new Error('Invalid relocated vector lines'); + } + await this.query( + 'MATCH (n:EmbeddingNodeReceipt {nodeId:$nodeId}) SET n.contentHash=$contentHash, n.sourceStartLine=$startLine, n.sourceEndLine=$endLine', + { nodeId: input.nodeId, startLine, endLine, contentHash }, + ); + }); + return { + ...previous, + contentHash, + sourceStartLine: startLine, + sourceEndLine: endLine, + }; + }, true); + } + + private async ensureSelectionTable(): Promise { + if (this.selectionReady) return; + // Auxiliary scheduling data leaves the vector/input schema unchanged. + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS EmbeddingSelection (nodeId STRING, job STRING, PRIMARY KEY(nodeId))', + ); + this.selectionReady = true; + } + + /** Mark only verified complete groups, before the unit checkpoint and cursor. + * Native selection rows replace an unbounded JS set of current node IDs. */ + async markSelected(job: string, ids: readonly string[]): Promise { + if ( + !/^[a-f0-9]{64}$/.test(job) || + !ids.length || + ids.length > 64 || + new Set(ids).size !== ids.length + ) + throw new Error('Invalid vector selection unit'); + for (const id of ids) assertValidGraphNodeId(id, 'Vector selection'); + await this.exclusive(async () => { + const receipts = await this.receipts(ids); + if ( + receipts.size !== ids.length || + [...receipts.values()].some((r) => !r.complete || r.chunks < 1) + ) + throw new Error('Cannot select an incomplete vector group'); + await this.ensureSelectionTable(); + await this.query( + 'UNWIND $ids AS id MERGE (s:EmbeddingSelection {nodeId:id}) SET s.job=$job', + { ids: [...ids], job }, + ); + }, true); + } + + /** Called only after the new manifest is fully acknowledged. Counts must + * match before deletion, and native joins keep pruning independent of JS RAM. */ + async pruneUnselected(job: string, totalNodes: number): Promise { + if (!/^[a-f0-9]{64}$/.test(job) || !nonnegative(totalNodes)) + throw new Error('Invalid vector selection coverage'); + await this.exclusive(async () => { + await this.ensureSelectionTable(); + const selected = await this.query( + 'MATCH (s:EmbeddingSelection) WHERE s.job=$job RETURN count(s) AS nodes', + { job }, + ); + if (count(selected[0].nodes) !== totalNodes) + throw new Error('Vector selection coverage is incomplete; no groups pruned'); + await this.transaction(async () => { + await this.query( + 'MATCH (e:CodeEmbedding) WHERE NOT EXISTS { MATCH (s:EmbeddingSelection) WHERE s.nodeId=e.nodeId AND s.job=$job } DELETE e', + { job }, + ); + await this.query( + 'MATCH (n:EmbeddingNodeReceipt) WHERE NOT EXISTS { MATCH (s:EmbeddingSelection) WHERE s.nodeId=n.nodeId AND s.job=$job } DELETE n', + { job }, + ); + await this.query('MATCH (s:EmbeddingSelection) WHERE s.job<>$job DELETE s', { job }); + }); + }, true); + } + + /** Replay replaces the entire old group, using PKs rather than repeated table scans. */ + async beginNode(nodeId: string, contentHash: string): Promise { + return this.beginNodes([{ nodeId, contentHash }]); + } + + /** The durable journal owns the three-attempt exclusion proof. Remove only + * those rejected nodes' partial rows and checkpoint before acknowledging. */ + async excludeNodes(ids: readonly string[]): Promise { + await this.beginNodes(ids.map((nodeId) => ({ nodeId, contentHash: 'input-rejected' }))); + await this.exclusive(async () => { + await this.query('UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) DELETE n', { + ids: [...ids], + }); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, true); + } + + async beginNodes(nodes: readonly VectorNodeInput[]): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector node unit'); + for (const node of nodes) { + assertValidGraphNodeId(node.nodeId, 'Vector store'); + if (typeof node.contentHash !== 'string' || !node.contentHash) + throw new Error('Missing embedding input hash'); + if ( + !nonnegative(node.sourceStartLine ?? 0) || + !nonnegative(node.sourceEndLine ?? 0) || + (node.sourceEndLine ?? 0) < (node.sourceStartLine ?? 0) + ) + throw new Error('Invalid embedding source lines'); + } + await this.exclusive(async () => { + const previous = await this.receipts(nodes.map((n) => n.nodeId)); + if (previous.size) { + // Clear the proof before deleting any row. The old high-water mark is + // retained until every old chunk has been removed, including on crash. + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=false', + { ids: [...previous.keys()] }, + ); + let ids: string[] = []; + const remove = async () => { + if (!ids.length) return; + await this.query('UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) DELETE e', { ids }); + ids = []; + }; + for (const receipt of previous.values()) + for (let i = 0; i < receipt.chunks; i++) { + ids.push(`${receipt.nodeId}:${i}`); + if (ids.length === BLOCK_ROWS) await remove(); + } + await remove(); + } + await this.query( + 'UNWIND range(1, size($ids)) AS i MERGE (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.chunks=0, n.complete=false, n.sourceStartLine=$firstLines[i], n.sourceEndLine=$lastLines[i]', + { + ids: nodes.map((node) => node.nodeId), + hashes: nodes.map((node) => node.contentHash), + firstLines: nodes.map((node) => node.sourceStartLine ?? 0), + lastLines: nodes.map((node) => node.sourceEndLine ?? 0), + }, + ); + }, true); + } + + async append( + nodeId: string, + contentHash: string, + chunks: readonly StoredVectorChunk[], + ): Promise { + return this.appendBatch(chunks.map((chunk) => ({ ...chunk, nodeId, contentHash }))); + } + + async appendBatch(chunks: readonly VectorWriteChunk[]): Promise { + if ( + !chunks.length || + chunks.length > BLOCK_ROWS || + chunks.reduce( + (bytes, chunk) => + bytes + + this.options.dimensions * 16 + + 128 + + 2 * (chunk.nodeId.length + chunk.contentHash.length), + 0, + ) > this.options.batchBytes + ) + throw new Error('Vector write exceeds its byte/row reservation'); + for (const chunk of chunks) { + assertValidGraphNodeId(chunk.nodeId, 'Vector store'); + if ( + !nonnegative(chunk.chunkIndex) || + chunk.chunkIndex >= MAX_CHUNKS || + !nonnegative(chunk.startLine) || + !nonnegative(chunk.endLine) || + chunk.endLine < chunk.startLine || + chunk.embedding.length !== this.options.dimensions || + chunk.embedding.some((n) => !Number.isFinite(n) || Math.abs(n) > 3.4028234663852886e38) + ) + throw new Error('Invalid embedding chunk identity, lines, or vector'); + } + await this.exclusive(async () => { + const ids = [...new Set(chunks.map((chunk) => chunk.nodeId))]; + if (ids.length > 64) throw new Error('Vector write crosses its node unit'); + const receipts = await this.receipts(ids); + for (const chunk of chunks) { + const receipt = receipts.get(chunk.nodeId); + if ( + !receipt || + receipt.complete || + receipt.contentHash !== chunk.contentHash || + chunk.chunkIndex !== receipt.chunks + ) + throw new Error('Vector chunks do not extend the active node contiguously'); + receipt.chunks++; + } + await this.transaction(async () => { + await this.query( + 'UNWIND range(1, size($ids)) AS i CREATE (:CodeEmbedding {id:$ids[i], nodeId:$nodeIds[i], chunkIndex:$ordinals[i], startLine:$firstLines[i], endLine:$lastLines[i], embedding:$vectors[i], contentHash:$hashes[i]})', + { + ids: chunks.map((chunk) => `${chunk.nodeId}:${chunk.chunkIndex}`), + nodeIds: chunks.map((chunk) => chunk.nodeId), + ordinals: chunks.map((chunk) => chunk.chunkIndex), + firstLines: chunks.map((chunk) => chunk.startLine), + lastLines: chunks.map((chunk) => chunk.endLine), + vectors: chunks.map((chunk) => chunk.embedding), + hashes: chunks.map((chunk) => chunk.contentHash), + }, + ); + await this.query( + 'UNWIND range(1, size($ids)) AS i MATCH (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.chunks=$chunks[i]', + { + ids: [...receipts.keys()], + chunks: [...receipts.values()].map((receipt) => receipt.chunks), + }, + ); + }); + }, true); + } + + private async verify(input: VectorNodeReceipt | readonly VectorNodeReceipt[]): Promise { + const receipts = 'nodeId' in input ? [input] : input; + const expected = new Map(); + const flush = async () => { + if (!expected.size) return; + const rows = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash', + { ids: [...expected.keys()] }, + ); + const seen = new Set(); + for (const row of rows) { + const entry = typeof row.id === 'string' ? expected.get(row.id) : undefined; + if ( + !entry || + row.nodeId !== entry.receipt.nodeId || + row.contentHash !== entry.receipt.contentHash || + count(row.chunkIndex) !== entry.ordinal || + seen.has(String(row.id)) + ) + throw new Error('Vector node has an invalid chunk set'); + seen.add(String(row.id)); + } + if (seen.size !== expected.size) throw new Error('Vector node is missing chunks'); + expected.clear(); + }; + // Most groups contain one chunk. Fill each bounded PK query across groups + // rather than paying a native round trip for every node in a unit. + for (const receipt of receipts) + for (let ordinal = 0; ordinal < receipt.chunks; ordinal++) { + expected.set(`${receipt.nodeId}:${ordinal}`, { receipt, ordinal }); + if (expected.size === BLOCK_ROWS) await flush(); + } + await flush(); + } + + async finishNode( + nodeId: string, + contentHash: string, + chunks: number, + ): Promise { + return (await this.finishNodes([{ nodeId, contentHash, chunks }]))[0]; + } + + async finishNodes( + nodes: readonly (VectorNodeInput & { chunks: number; inputsDigest?: string })[], + ): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector completion unit'); + return this.exclusive(async () => { + const receipts = await this.receipts(nodes.map((n) => n.nodeId)); + const verified: VectorNodeReceipt[] = []; + for (const node of nodes) { + const receipt = receipts.get(node.nodeId); + if ( + !receipt || + receipt.contentHash !== node.contentHash || + receipt.chunks !== node.chunks || + node.chunks < 1 || + (node.inputsDigest !== undefined && + (!/^[a-f0-9]{40}$/.test(node.inputsDigest) || !/^[a-f0-9]{40}$/.test(node.contentHash))) + ) + throw new Error('Vector node coverage does not match its generated chunks'); + verified.push(receipt); + } + await this.verify(verified); + const finalize = async () => { + let ids: string[] = []; + let hashes: string[] = []; + const flush = async () => { + if (!ids.length) return; + await this.query( + 'UNWIND range(1, size($ids)) AS i MATCH (e:CodeEmbedding {id:$ids[i]}) SET e.contentHash=$hashes[i]', + { ids, hashes }, + ); + ids = []; + hashes = []; + }; + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i]; + if (!node.inputsDigest) continue; + const contentHash = `${node.contentHash}:${node.inputsDigest}`; + for (let ordinal = 0; ordinal < node.chunks; ordinal++) { + ids.push(`${node.nodeId}:${ordinal}`); + hashes.push(contentHash); + if (ids.length === BLOCK_ROWS) await flush(); + } + verified[i] = { ...verified[i], contentHash }; + } + await flush(); + await this.query( + 'UNWIND range(1, size($ids)) AS i MATCH (n:EmbeddingNodeReceipt {nodeId:$ids[i]}) SET n.contentHash=$hashes[i], n.complete=true', + { + ids: verified.map((receipt) => receipt.nodeId), + hashes: verified.map((receipt) => receipt.contentHash), + }, + ); + }; + if (nodes.some((node) => node.inputsDigest)) await this.transaction(finalize); + else + await this.query( + 'UNWIND $ids AS id MATCH (n:EmbeddingNodeReceipt {nodeId:id}) SET n.complete=true', + { ids: nodes.map((node) => node.nodeId) }, + ); + return verified.map((receipt) => ({ ...receipt, complete: true })); + }, true); + } + + async reusable( + nodeId: string, + contentHash: string, + options?: Pick, + ): Promise { + const matchesHash = options?.matchesHash; + return this.exclusive(async () => { + const receipt = await this.receipt(nodeId); + if ( + !receipt?.complete || + receipt.chunks < 1 || + (receipt.contentHash !== contentHash && !(await matchesHash?.(receipt.contentHash))) + ) + return null; + await this.verify(receipt); + return receipt; + }); + } + + /** One bounded metadata pass builds PK lookups beside the pending writer. + * No extra native pool, vector copy, or corpus-sized JS receipt map. A failed + * build leaves complete=false; retry replaces only this auxiliary index. */ + async prepareLegacyReuseIndex( + source: string, + groups: () => AsyncIterable, + excluded: () => AsyncIterable | Iterable, + signal?: AbortSignal, + ): Promise { + if (!/^[a-f0-9]{64}$/.test(source)) throw new Error('Invalid legacy source identity'); + await this.exclusive(async () => { + signal?.throwIfAborted(); + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS LegacyEmbeddingGroup (nodeId STRING, contentHash STRING, chunks INT64, sourceStartLine INT64, sourceEndLine INT64, PRIMARY KEY(nodeId))', + ); + await this.query( + 'CREATE NODE TABLE IF NOT EXISTS LegacyEmbeddingIndex (id STRING, source STRING, complete BOOL, PRIMARY KEY(id))', + ); + const state = await this.query( + "MATCH (s:LegacyEmbeddingIndex {id:'source'}) RETURN s.source AS source, s.complete AS complete", + ); + if (state.length === 1 && state[0].source === source && state[0].complete === true) return; + await this.query( + "MERGE (s:LegacyEmbeddingIndex {id:'source'}) SET s.source=$source, s.complete=false", + { source }, + ); + await this.query('MATCH (g:LegacyEmbeddingGroup) DELETE g'); + let batch: LegacyVectorGroup[] = []; + let bytes = 0; + const flush = async () => { + if (!batch.length) return; + signal?.throwIfAborted(); + await this.query( + 'UNWIND range(1, size($ids)) AS i CREATE (:LegacyEmbeddingGroup {nodeId:$ids[i], contentHash:$hashes[i], chunks:$chunks[i], sourceStartLine:$firstLines[i], sourceEndLine:$lastLines[i]})', + { + ids: batch.map((g) => g.nodeId), + hashes: batch.map((g) => g.contentHash), + chunks: batch.map((g) => g.chunks), + firstLines: batch.map((g) => g.sourceStartLine), + lastLines: batch.map((g) => g.sourceEndLine), + }, + ); + batch = []; + bytes = 0; + }; + for await (const group of groups()) { + signal?.throwIfAborted(); + assertValidGraphNodeId(group.nodeId, 'Legacy vector group'); + const size = 256 + 8 * (group.nodeId.length + group.contentHash.length); + if ( + !group.contentHash || + !nonnegative(group.chunks) || + !group.chunks || + group.chunks > MAX_CHUNKS || + !nonnegative(group.sourceStartLine) || + !nonnegative(group.sourceEndLine) || + group.sourceEndLine < group.sourceStartLine || + size > this.options.batchBytes + ) + throw new Error('Invalid or oversized legacy group receipt'); + if (batch.length === 64 || bytes + size > this.options.batchBytes) await flush(); + batch.push(group); + bytes += size; + } + await flush(); + let unsafe: string[] = []; + const remove = async () => { + if (!unsafe.length) return; + signal?.throwIfAborted(); + await this.query('UNWIND $ids AS id MATCH (g:LegacyEmbeddingGroup {nodeId:id}) DELETE g', { + ids: unsafe, + }); + unsafe = []; + bytes = 0; + }; + for await (const id of excluded()) { + signal?.throwIfAborted(); + assertValidGraphNodeId(id, 'Legacy unsafe window'); + const size = 256 + 8 * id.length; + if (size > this.options.batchBytes) throw new Error('Oversized legacy exclusion'); + if (unsafe.length === 64 || bytes + size > this.options.batchBytes) await remove(); + unsafe.push(id); + bytes += size; + } + await remove(); + signal?.throwIfAborted(); + await this.query("MATCH (s:LegacyEmbeddingIndex {id:'source'}) SET s.complete=true"); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, true); + } + + async legacyReuseGroup(source: string, nodeId: string): Promise { + if (!/^[a-f0-9]{64}$/.test(source)) throw new Error('Invalid legacy source identity'); + assertValidGraphNodeId(nodeId, 'Legacy vector group'); + return this.exclusive(async () => { + const rows = await this.query( + "MATCH (s:LegacyEmbeddingIndex {id:'source'}), (g:LegacyEmbeddingGroup {nodeId:$nodeId}) WHERE s.source=$source AND s.complete=true RETURN g.contentHash AS contentHash, g.chunks AS chunks, g.sourceStartLine AS sourceStartLine, g.sourceEndLine AS sourceEndLine", + { source, nodeId }, + ); + if (!rows.length) return null; + if (rows.length !== 1 || typeof rows[0].contentHash !== 'string' || !rows[0].contentHash) + throw new Error('Corrupt legacy group receipt'); + const group = { + nodeId, + contentHash: rows[0].contentHash, + chunks: count(rows[0].chunks), + sourceStartLine: count(rows[0].sourceStartLine), + sourceEndLine: count(rows[0].sourceEndLine), + }; + if (!group.chunks || group.chunks > MAX_CHUNKS || group.sourceEndLine < group.sourceStartLine) + throw new Error('Corrupt legacy group receipt'); + return group; + }); + } + + /** Same immutable job identity: completed groups need no source/model work. */ + async completedNodes(ids: readonly string[]): Promise> { + if (ids.length > 64 || new Set(ids).size !== ids.length) + throw new Error('Invalid vector receipt unit'); + return this.exclusive(async () => { + const receipts = await this.receipts(ids); + for (const [id, receipt] of receipts) { + if (!receipt.complete || receipt.chunks < 1) receipts.delete(id); + else await this.verify(receipt); + } + return receipts; + }); + } + + /** Bounded copy from a compatible complete artifact; never materializes a node's vectors. */ + async readChunks( + receipt: VectorNodeReceipt, + start: number, + length: number, + ): Promise { + if ( + !nonnegative(start) || + !nonnegative(length) || + !length || + length > BLOCK_ROWS || + start + length > receipt.chunks || + length * (this.options.dimensions * 16 + 128) > this.options.batchBytes + ) + throw new Error('Invalid vector read batch'); + return this.exclusive(async () => { + const rows = await this.query( + 'UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash', + { ids: Array.from({ length }, (_, i) => `${receipt.nodeId}:${start + i}`) }, + ); + if (rows.length !== length) throw new Error('Incomplete vector copy'); + const chunks = rows + .map((row) => { + if ( + row.contentHash !== receipt.contentHash || + !Array.isArray(row.embedding) || + row.embedding.length !== this.options.dimensions + ) + throw new Error('Invalid vector copy'); + return { + chunkIndex: count(row.chunkIndex), + startLine: count(row.startLine), + endLine: count(row.endLine), + embedding: row.embedding as number[], + }; + }) + .sort((a, b) => a.chunkIndex - b.chunkIndex); + if (chunks.some((chunk, i) => chunk.chunkIndex !== start + i)) + throw new Error('Invalid vector copy sequence'); + return chunks; + }); + } + + /** Durability must precede the filesystem journal's acknowledgement. */ + async checkpointUnit(nodes: readonly VectorNodeReceipt[]): Promise { + if ( + !nodes.length || + nodes.length > 64 || + new Set(nodes.map((n) => n.nodeId)).size !== nodes.length + ) + throw new Error('Invalid vector durability unit'); + await this.exclusive(async () => { + const receipts = await this.receipts(nodes.map((n) => n.nodeId)); + for (const node of nodes) { + const stored = receipts.get(node.nodeId); + if ( + !stored?.complete || + stored.contentHash !== node.contentHash || + stored.chunks !== node.chunks || + stored.chunks < 1 + ) + throw new Error('Cannot acknowledge incomplete vector unit'); + } + // An already sealed artifact has a durable final checkpoint. Recovery may + // rebuild a damaged filesystem cursor using its verified complete groups. + if (this.sealed) return; + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + }, !this.sealed); + } + + /** Runs only once all manifest and failed units have complete coverage. */ + async seal(totalNodes: number, totalChunks: number): Promise<'hnsw' | 'exact'> { + if (!nonnegative(totalNodes) || !nonnegative(totalChunks)) + throw new Error('Invalid vector coverage totals'); + return this.exclusive(async () => { + const nodes = await this.query( + 'MATCH (n:EmbeddingNodeReceipt) RETURN count(n) AS nodes, sum(n.chunks) AS chunks, sum(CASE WHEN n.complete THEN 0 ELSE 1 END) AS pending', + ); + const vectors = await this.query('MATCH (e:CodeEmbedding) RETURN count(e) AS chunks'); + if ( + count(nodes[0].nodes) !== totalNodes || + count(nodes[0].chunks ?? 0) !== totalChunks || + count(nodes[0].pending ?? 0) !== 0 || + count(vectors[0].chunks) !== totalChunks + ) + throw new Error('Vector artifact coverage is incomplete'); + if (this.sealed) { + const rows = await this.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.indexKind AS kind", + ); + if (rows[0]?.kind !== 'hnsw' && rows[0]?.kind !== 'exact') + throw new Error('Invalid vector index receipt'); + if (rows[0].kind === 'hnsw') { + const manager = new ExtensionManager(); + if ( + !(await manager.ensure((sql) => this.query(sql), 'vector', 'vector publication', { + policy: 'load-only', + })) + ) + throw new Error('Cannot verify the published vector index'); + await this.verifyVectorIndex(); + } + return rows[0].kind; + } + const manager = new ExtensionManager(); + let kind: 'hnsw' | 'exact' = 'exact'; + if ( + totalChunks && + (await manager.ensure((sql) => this.query(sql), 'vector', 'vector search')) + ) { + try { + await this.query( + "CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')", + ); + kind = 'hnsw'; + } catch (error) { + // A complete artifact may be searched exactly if optional ANN fails. + // Native corruption/durability errors still surface at checkpoint. + if (/already exists/i.test(String(error))) kind = 'hnsw'; + else + logger.warn( + { err: error, chunks: totalChunks, bufferPoolBytes: this.options.bufferPoolBytes }, + 'Vector index build failed; completed vectors retained', + ); + } + } + if (kind === 'hnsw') { + try { + await this.verifyVectorIndex(); + } catch (error) { + logger.warn( + { err: error }, + 'Vector index verification failed; dropping the unpublished index for retry', + ); + await this.query("CALL DROP_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx')"); + kind = 'exact'; + } + } + // Large exact scans are deliberately unavailable to readers. Keep this + // complete work unpublished and writable so a later run can retry ANN + // without a single new provider request. + if (kind === 'exact' && totalChunks > getExactScanLimit()) { + logger.warn( + { chunks: totalChunks, bufferPoolBytes: this.options.bufferPoolBytes }, + 'Vector index is pending beyond the exact-scan limit. Increase --resource-budget and retry; completed vectors will be reused.', + ); + return kind; + } + await this.query( + "MATCH (a:EmbeddingArtifact {id:'artifact'}) SET a.complete=true, a.indexKind=$kind", + { kind }, + ); + try { + await this.query('CHECKPOINT'); + } catch (error) { + this.poisoned = true; + throw error; + } + this.sealed = true; + return kind; + }, !this.sealed); + } + + private async verifyVectorIndex(): Promise { + // determinism: probe — any stored vector can verify index availability; its identity does not affect publication. + const sample = await this.query( + 'MATCH (e:CodeEmbedding) RETURN e.embedding AS embedding LIMIT 1', + ); + if (sample.length !== 1 || !Array.isArray(sample[0].embedding)) + throw new Error('Vector index has no probe row'); + const hits = await this.query( + "CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', $vector, 1) YIELD node AS emb, distance RETURN emb.id AS id", + { vector: sample[0].embedding as number[] }, + ); + if (hits.length !== 1 || typeof hits[0].id !== 'string') + throw new Error('Vector index cannot retrieve a stored row'); + } + + async close(): Promise { + if (this.closing) return this.closing; + if (this.busy) throw new Error('Cannot close an active vector operation'); + this.closed = true; + this.prepared.clear(); + this.closing = closeLbugConnection(this.handle, { + strict: true, + databasePath: path.join(this.options.directory, 'vectors'), + }); + try { + await this.closing; + } catch (error) { + this.closing = undefined; + throw error; + } + } +} diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3d6d53a5c..3d7a0de79 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -11,7 +11,7 @@ import { detectGraphWriteCollapse, - getIndexIncompleteReasons, + getGraphIncompleteReasons, type GraphWriteCollapseVerdict, } from './index-freshness.js'; import { @@ -38,15 +38,12 @@ import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js'; import path from 'path'; import fs from 'fs/promises'; import { constants as fsConstants, existsSync } from 'node:fs'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import { retryRename } from '../storage/fs-atomic.js'; import { assertLbugPublicationReady, consolidateLbugForPublication } from './lbug/publication.js'; -import { - acquireIndexLock, - requireExclusiveIndexLock, - sweepStagingArtifacts, -} from '../storage/index-lock.js'; +import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { resolveEmbeddingRecovery } from '../storage/embedding-recovery.js'; import { invalidateNodeWorkspacePackages } from './ingestion/languages/typescript/workspace-packages.js'; import { @@ -54,7 +51,6 @@ import { summarizeNameFallback, countCallsByLanguage, } from './ingestion/scope-resolution/name-fallback-summary.js'; -import { runPipelineFromRepo } from './ingestion/pipeline.js'; import type { RouteHookConfig } from './ingestion/language-provider.js'; import { routeHookConfigKey } from './ingestion/route-hook-config.js'; import { @@ -65,6 +61,11 @@ import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/und import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import { summarizeUnresolvedOverloads } from './ingestion/scope-resolution/unresolved-overloads.js'; import type { KnowledgeGraph } from './graph/types.js'; +import { + publishAnalysisAfterProcessExit, + type AnalysisPhaseTarget, + type PreparedAnalysisHandoff, +} from './analysis-phase-supervisor.js'; import { reconcileGraphNodeIdentities } from './incremental/write-reconciliation.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { @@ -76,6 +77,7 @@ import { forEachQueryRow, executeWithReusedStatement, closeLbug, + drainAnalysisNativeResources, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, @@ -174,6 +176,8 @@ import { loadMeta, ensureGitNexusIgnored, registerRepo, + readRegistry, + findRegistryEntryByRepoPath, setShareOptOut, adoptFlatBranchLabel, isReadOnlyFilesystemError, @@ -199,6 +203,7 @@ import { import { isSharedStoreDisabled, resolveGraphPath, + resolveGraphWritePath, resolveSharedStore, storeRootOfCheckoutSlot, type SharedStoreLayout, @@ -206,7 +211,6 @@ import { import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { ensurePrivateSharedGraph, - listStoreMetaRoots, leaveSharedStore, optedInSlotToLeave, registerLeftStore, @@ -214,7 +218,17 @@ import { resolveOptedInStore, seedSharedSlot, } from './shared-store-analyze.js'; -import { withStoreLock } from '../storage/shared-store-lifecycle.js'; +import { + publishAnalysisParseCache, + loadAnalysisParseCache, + stageAnalysisParseCache, +} from './analysis-parse-cache.js'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from './analysis-publication.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js'; import { DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, @@ -242,7 +256,6 @@ import { type PersistedSpringConfigConsumerRow, } from './incremental/spring-config-drift.js'; import { shadowCandidatesFor } from './incremental/shadow-candidates.js'; -import { collectResolutionGroupFiles } from './incremental/resolution-groups.js'; import { shouldEscalateIncrementalWrite } from './incremental/escalation-gate.js'; import { ftsTablesAmong, @@ -266,14 +279,11 @@ import { } from './parse-cache-identity.js'; import { loadParseCache, - saveParseCache, - pruneCache, createColdParseRebuildDir, getColdParseRebuildDir, emptyParseCache, forgetCreatedParseCacheDir, } from '../storage/parse-cache.js'; -import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js'; import { getCurrentCheckout, getRemoteUrl, @@ -457,11 +467,27 @@ function collectSpringBeanDeclarationDriftFiles( // --------------------------------------------------------------------------- export interface AnalyzeCallbacks { - onProgress: (phase: string, percent: number, message: string) => void; + /** Process-local cancellation; never serialized through options/IPC. */ + signal?: AbortSignal; + /** Exit-only frontends admit their supervisor before opening a graph writer. */ + onGraphWork?: () => void | Promise; + /** Let a serving parent refresh graph readers before paid embedding work. */ + onGraphPublished?: () => Promise; + onProgress: ( + phase: string, + percent: number, + message: string, + details?: import('./analysis-progress.js').AnalysisPhaseProgress, + ) => void; onLog?: (message: string) => void; } export interface AnalyzeOptions { + /** Internal phase coordination. Entry points activate this together with + * the published embedding continuation; never a user-facing mode switch. */ + graphPublication?: 'auto' | 'inline' | 'after-exit'; + /** Frozen per-invocation allocations; entrypoints resolve these before destructive work. */ + resources?: import('./analysis-resource-budget.js').AnalysisResourceBudget; /** * Rebuild the graph and FTS. Parser output is still reused from the * content-addressed parse cache unless `useParseCache` is false. @@ -789,6 +815,8 @@ export function analyzeFailureMayHaveMutatedLiveIndex(error: unknown): boolean { } export interface AnalyzeResult { + /** Exit-only graph phase awaiting its supervisor's verified publication. */ + preparedGraph?: PreparedAnalysisHandoff; repoName: string; repoPath: string; /** The exact storage slot selected and validated for this analysis. */ @@ -1281,6 +1309,35 @@ interface WriteTarget { sharedStore?: SharedStoreLayout; } +/** Adoption must not move the checkout away from its unfinished private work. + * Only inspected during a local-to-shared transition; established shared slots + * keep their existing fast path. Branch jobs belong to the same local root. */ +async function hasRetainedAnalysisWork(root: string): Promise { + const pending = async (slot: string): Promise => { + try { + await fs.lstat(path.join(slot, 'analysis-publication.json')); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const meta = await loadMeta(slot); + return Boolean(meta?.incrementalInProgress || meta?.embeddingJob || meta?.embeddingCheckpoint); + }; + if (await pending(root)) return true; + let branches; + try { + branches = await fs.opendir(path.join(root, 'branches')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } + for await (const branch of branches) { + if (branch.isDirectory() && (await pending(path.join(root, 'branches', branch.name)))) + return true; + } + return false; +} + /** * Resolve which storage slot this analyze writes to, including branch * placement (#2106/#2354). Extracted from the top of the pipeline so the lock @@ -1313,15 +1370,29 @@ async function resolveWriteTarget(repoPath: string, options: AnalyzeOptions): Pr ); } const sharingOff = options.noShare || isSharedStoreDisabled(); - const sharedStore = sharingOff + let sharedStore = sharingOff ? undefined : (resolveSharedStore(repoPath) ?? (await resolveOptedInStore(repoPath, options.shareWith))); + const previousStorage = resolveStoragePath(repoPath); + let retainedStorage: string | undefined; + if ( + sharedStore && + path.resolve(previousStorage) !== path.resolve(sharedStore.checkoutSlot) && + !storeRootOfCheckoutSlot(previousStorage) && + (await hasRetainedAnalysisWork(previousStorage)) + ) { + // Re-resolved after ordinary lock acquisition as well: if the previous + // writer completes while we wait, adoption can proceed on the next lock. + retainedStorage = previousStorage; + sharedStore = undefined; + } // A checkout still registered in a store after sharing was turned off // indexes into its own `.gitnexus` again; its slot is left for `clean --gc`. - const leavingStore = - !sharedStore && storeRootOfCheckoutSlot(resolveStoragePath(repoPath)) !== null; + const leavingStore = !sharedStore && storeRootOfCheckoutSlot(previousStorage) !== null; const explicitStorage = - sharedStore?.checkoutSlot ?? (leavingStore ? defaultStoragePath(repoPath) : undefined); + retainedStorage ?? + sharedStore?.checkoutSlot ?? + (leavingStore ? defaultStoragePath(repoPath) : undefined); const storagePath = explicitStorage ? await requireRegisteredStoragePath( { path: repoPath, storagePath: explicitStorage }, @@ -1357,10 +1428,10 @@ async function resolveWriteTarget(repoPath: string, options: AnalyzeOptions): Pr : {}; const paths = getStoragePaths(repoPath, placement.branch, storagePath); const { metaPath } = paths; - // Analyze always writes a store slot's own graph; a recorded `graphPath` - // (an immutable commit graph) only redirects readers. + // Owned generation files stay writable in place. Only an immutable shared + // commit pointer resolves to a fresh slot-local write target. const lbugPath = storeRootOfCheckoutSlot(storagePath) - ? path.join(path.dirname(metaPath), LBUG_DIRECTORY) + ? resolveGraphWritePath(path.dirname(metaPath)) : paths.lbugPath; return { storagePath, @@ -1395,6 +1466,240 @@ interface ParseIdentityRun { storagePath?: string; } +async function retainsPreparedGraph(metadataDirectory: string): Promise { + return fs.lstat(path.join(metadataDirectory, 'analysis-publication.json')).then( + () => true, + (error: NodeJS.ErrnoException) => error.code !== 'ENOENT', + ); +} + +/** Preview the registry's existing name precedence without advancing freshness. + * The continuation checks it again under the slot lock before publication. */ +async function preparedAnalysisName(repoPath: string, options: AnalyzeOptions): Promise { + if (options.registryName !== undefined) return options.registryName; + const existing = findRegistryEntryByRepoPath(await readRegistry(), repoPath); + const inferred = getInferredRepoName(repoPath); + if ( + existing && + existing.name !== path.basename(path.resolve(existing.path)) && + existing.name !== inferred && + existing.name !== path.basename(resolveRepoIdentityRoot(existing.path)) + ) + return existing.name; + return inferred ?? path.basename(resolveRepoIdentityRoot(repoPath)); +} + +async function recoverPreparedGraph( + target: WriteTarget, + repoPath: string, + options: AnalyzeOptions, + log: (message: string) => void, +): Promise { + const prepared = await retainsPreparedGraph(target.metaDir); + if (prepared && !(await readPreparedAnalysis(target.metaDir))) return undefined; + // Publication removes the graph receipt before the optional cache is merged. + // Recover that crash window using only the currently published generation. + const alreadyPublished = prepared ? undefined : await loadMeta(target.metaDir); + let pendingCache: Awaited>; + if (!prepared) { + if (!alreadyPublished?.graphGeneration || alreadyPublished.incrementalInProgress) + return undefined; + try { + pendingCache = await loadAnalysisParseCache(target.metaDir, alreadyPublished); + } catch (error) { + log(`Warning: could not recover the prepared parse cache (${(error as Error).message}).`); + return undefined; + } + if (!pendingCache) return undefined; + } + if (!options.resources) throw new Error('Graph publication requires admitted resources'); + const metadata = + alreadyPublished ?? + (await publishPreparedAnalysis(target.metaDir, { + saveMetadata: (meta) => saveMeta(target.metaDir, meta), + verify: async (file, meta) => { + if (meta.repoPath !== repoPath) + throw new Error('Prepared graph belongs to another checkout'); + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + await verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: options.resources!.parse.graphPoolBytes, + threads: options.resources!.parse.nativeThreads, + }); + }, + })); + if (!metadata) throw new Error('Prepared graph disappeared under its writer lock'); + await registerRepo(repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: target.placement.branch, + storagePath: target.storagePath, + }); + try { + const cache = pendingCache ?? (await loadAnalysisParseCache(target.metaDir, metadata)); + if (cache) + await publishAnalysisParseCache(cache, { + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + cacheRoot: target.sharedStore?.cachesDir ?? target.storagePath, + publishedGeneration: metadata.graphGeneration?.id, + sharedStore: target.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + } catch (error) { + log( + `Warning: could not publish the prepared parse cache (${(error as Error).message}); continuing.`, + ); + } + return metadata; +} + +/** Complete an exit-only writer's exact attempt, without invoking the parser or + * regenerating context/skills. The launcher must wait for actual process exit + * before starting this continuation in its separately admitted process. */ +export async function continuePreparedAnalysis( + handoff: PreparedAnalysisHandoff, + writerPid: number, + options: AnalyzeOptions, + callbacks: AnalyzeCallbacks, + signal?: AbortSignal, +): Promise { + const resources = options.resources; + if (!resources) throw new Error('Graph continuation requires admitted resources'); + const log = (message: string) => callbacks.onLog?.(stripControlCharacters(message)); + // Publication resolves the target before locking and again under the lock. + // Retain that verified target for its callbacks; an extra snapshot here is + // overwritten before use and adds synchronous Git probes to every handoff. + let target: Awaited>; + const resolveTarget = async (): Promise => { + target = await resolveWriteTarget(handoff.repoPath, options); + return { + repoPath: handoff.repoPath, + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + repoName: await preparedAnalysisName(handoff.repoPath, options), + commit: target.currentCommit, + }; + }; + const finalize = async (metadata: RepoMeta): Promise => { + const prior = options.noShare ? await optedInSlotToLeave(handoff.repoPath) : undefined; + await registerRepo(handoff.repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: target.placement.branch, + storagePath: target.storagePath, + }); + if (!target.placement.branch && target.branchLabel) { + try { + await adoptFlatBranchLabel(handoff.repoPath, target.branchLabel, target.storagePath); + } catch (error) { + log(`Warning: could not sync the workspace branch label (${(error as Error).message}).`); + } + } + if (target.sharedStore && !target.placement.branch) { + await publishSharedGraph(target.sharedStore, handoff.repoPath, target.currentCommit, log, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + }); + } else if (!target.sharedStore) { + if (prior) await leaveSharedStore(handoff.repoPath, prior, target.storagePath, log); + else await registerLeftStore(handoff.repoPath, target.storagePath); + } + if (options.noShare) await setShareOptOut(handoff.repoPath, true); + else if (options.shareWith) await setShareOptOut(handoff.repoPath, false); + return (await loadMeta(target.metaDir)) ?? metadata; + }; + let metadata = await publishAnalysisAfterProcessExit(handoff, writerPid, { + resolveTarget, + saveMetadata: (meta) => saveMeta(target.metaDir, meta), + verify: async (file, meta) => { + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + await verifyPublishedAnalysisGraph(file, meta, { + bufferPoolBytes: resources.embedding.graphPoolBytes, + threads: resources.embedding.nativeThreads, + }); + }, + finalizeGraph: async (meta) => { + const published = await finalize(meta); + try { + const cache = await loadAnalysisParseCache(target.metaDir, published); + if (cache) + await publishAnalysisParseCache(cache, { + storagePath: target.storagePath, + metadataDirectory: target.metaDir, + cacheRoot: target.sharedStore?.cachesDir ?? target.storagePath, + publishedGeneration: published.graphGeneration?.id, + sharedStore: target.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + await removeColdParseRebuildDir(getColdParseRebuildDir(target.metaDir), false); + } catch (error) { + log(`Warning: could not publish the prepared parse cache (${(error as Error).message}).`); + } + callbacks.onProgress('graph-ready', 90, 'Graph published', { graphPhase: 'ready' }); + if ( + !_deriveEmbeddingMode(options, published.stats?.embeddings ?? 0).shouldGenerateEmbeddings && + !published.embeddingJob && + !published.embeddingCheckpoint + ) + await callbacks.onGraphPublished?.(); + return published; + }, + signal, + log, + onWaitStart: () => callbacks.onProgress('lock', 0, 'Waiting to publish the prepared graph…'), + }); + const generate = _deriveEmbeddingMode( + options, + metadata.stats?.embeddings ?? 0, + ).shouldGenerateEmbeddings; + if (generate || metadata.embeddingJob || metadata.embeddingCheckpoint) { + signal?.throwIfAborted(); + const lock = await acquireAnalysisSlotLock(handoff.metadataDirectory, { sweep: false, log }); + try { + requireExclusiveIndexLock(lock, 'Embedding continuation requires exclusive index ownership'); + const current = await resolveTarget(); + for (const key of [ + 'repoPath', + 'storagePath', + 'metadataDirectory', + 'repoName', + 'commit', + ] as const) + if (current[key] !== handoff[key]) + throw new Error('Analysis target changed before embedding; retry analysis.'); + const { runAnalysisEmbeddingPhase } = await import('./analysis-embedding-phase.js'); + metadata = await runAnalysisEmbeddingPhase({ + metadataDirectory: target.metaDir, + graphPath: resolveGraphPath(target.metaDir), + resources, + callbacks, + nodeLimit: options.embeddingsNodeLimit, + expectedGeneration: handoff.publicationId, + signal, + }); + metadata = await finalize(metadata); + // The graph writer deliberately retained staging files across its exit. + // Reclaim them only after the replacement vectors are published, while + // still holding the ordinary slot lock. + sweepStagingArtifacts(target.metaDir, log); + } finally { + lock.release(); + } + } + callbacks.onProgress('done', 100, 'Done', { graphPhase: 'ready' }); + return metadata; +} + function effectiveParseCacheIdentity( identity: ParseCacheIdentity, options: Pick, @@ -1432,6 +1737,13 @@ export async function runFullAnalysis( callbacks: AnalyzeCallbacks, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { + callbacks.signal?.throwIfAborted(); + if (options.graphPublication && !options.resources) + throw new Error('Graph publication requires admitted resources'); + if (options.graphPublication === 'after-exit' && !options.skipNativeCloseOnExit) + throw new Error('Deferred publication requires an exit-only graph writer'); + if (options.graphPublication === 'inline' && options.skipNativeCloseOnExit) + throw new Error('Inline publication requires physical native release'); // Validate operator-provided FTS config before anything else — a typo fails // here in ms, without taking the lock. (createSearchFTSIndexes reuses the // cached value via getSearchFTSStemmer.) @@ -1463,7 +1775,8 @@ export async function runFullAnalysis( }; let writeTarget = await resolveWriteTarget(repoPath, options); - let lock = await acquireIndexLock(writeTarget.metaDir, acquireOpts); + let lock = await acquireAnalysisSlotLock(writeTarget.metaDir, acquireOpts); + let retainParseStage = false; return withEmbeddingSpillScope(async () => { try { requireExclusiveIndexLock( @@ -1498,7 +1811,7 @@ export async function runFullAnalysis( ); lock.release(); writeTarget = fresh; - lock = await acquireIndexLock(fresh.metaDir, acquireOpts); + lock = await acquireAnalysisSlotLock(fresh.metaDir, acquireOpts); requireExclusiveIndexLock( lock, `Cannot acquire the index lock at ${fresh.metaDir}; refusing an unlocked analysis.`, @@ -1514,10 +1827,37 @@ export async function runFullAnalysis( `Warning: checkout "${formatRejectedBranchForLog(writeTarget.rejectedDetectedBranch)}" is not a usable index label; continuing.`, ); } + if (options.graphPublication === 'auto') { + const prior = await loadMeta(writeTarget.metaDir); + const separate = Boolean( + (await retainsPreparedGraph(writeTarget.metaDir)) || + prior?.graphGeneration || + prior?.embeddingJob || + (!options.dropEmbeddings && prior?.embeddingCheckpoint) || + _deriveEmbeddingMode(options, prior?.stats?.embeddings ?? 0).shouldGenerateEmbeddings, + ); + options = { + ...options, + graphPublication: separate + ? options.skipNativeCloseOnExit + ? 'after-exit' + : 'inline' + : undefined, + }; + } // Only this canonical slot is locked. Its deterministic parser stage // can only belong to an interrupted predecessor; sibling stages live // under their own meta directories and must remain untouched. Reclaim // before freshness/FTS-only early returns as well as parsing runs. + const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined; + if (options.graphPublication) { + const recovered = await recoverPreparedGraph(writeTarget, repoPath, options, log); + if (recovered?.graphPath) + writeTarget = { + ...writeTarget, + lbugPath: resolveGraphWritePath(writeTarget.metaDir), + }; + } await removeColdParseRebuildDir(getColdParseRebuildDir(writeTarget.metaDir), false); const flatShared = writeTarget.placement.branch ? undefined : writeTarget.sharedStore; if (flatShared) { @@ -1538,7 +1878,6 @@ export async function runFullAnalysis( }); } sweepStagingArtifacts(writeTarget.metaDir, log); - const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined; const result = await runFullAnalysisInner( repoPath, options, @@ -1548,6 +1887,10 @@ export async function runFullAnalysis( parseRun, runnerIdentityAtBootstrap, ); + if (result.preparedGraph) { + retainParseStage = true; + return result; + } if (flatShared) { await publishSharedGraph(flatShared, repoPath, writeTarget.currentCommit, log, { name: options.registryName, @@ -1568,11 +1911,83 @@ export async function runFullAnalysis( // A clone that left stays out of sibling stores until `--share-with`. if (options.noShare) await setShareOptOut(repoPath, true); else if (options.shareWith) await setShareOptOut(repoPath, false); + if ( + options.graphPublication === 'inline' || + (options.graphPublication === 'after-exit' && result.alreadyUpToDate) + ) { + let published = await loadMeta(writeTarget.metaDir); + const generate = _deriveEmbeddingMode( + options, + published?.stats?.embeddings ?? 0, + ).shouldGenerateEmbeddings; + if (published && (generate || published.embeddingJob || published.embeddingCheckpoint)) { + if (flatShared && !published.graphGeneration) { + if (!(await ensurePrivateSharedGraph(writeTarget.metaDir, log))) + throw new Error('The shared graph is unavailable; re-run analyze to rebuild it.'); + published = (await loadMeta(writeTarget.metaDir)) ?? published; + } + const { runAnalysisEmbeddingPhase } = await import('./analysis-embedding-phase.js'); + const metadata = await runAnalysisEmbeddingPhase({ + metadataDirectory: writeTarget.metaDir, + graphPath: resolveGraphPath(writeTarget.metaDir, published), + resources: options.resources!, + callbacks, + nodeLimit: options.embeddingsNodeLimit, + expectedGeneration: published.graphGeneration?.id, + signal: callbacks.signal, + }); + // A matching completed artifact performs no writes. Preserve the + // graph's no-op result and avoid republishing identical registry and + // shared-store metadata on every cached embedding invocation. + const unchangedVectors = + result.alreadyUpToDate && + !published.embeddingJob && + !published.embeddingCheckpoint && + published.vectorGeneration && + metadata.vectorGeneration?.identity === published.vectorGeneration.identity; + if (!unchangedVectors) { + await registerRepo(repoPath, metadata, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: writeTarget.placement.branch, + storagePath: writeTarget.storagePath, + }); + if (flatShared) + await publishSharedGraph(flatShared, repoPath, writeTarget.currentCommit, log, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + }); + result.alreadyUpToDate = false; + } + result.stats = metadata.stats ?? {}; + callbacks.onProgress('done', 100, 'Done', { + graphPhase: 'ready', + embeddingProgress: { + phase: + metadata.vectorGeneration?.graphGeneration === metadata.graphGeneration?.id && + !metadata.embeddingJob && + metadata.vectorGeneration + ? 'ready' + : metadata.stats?.embeddings + ? 'preserved' + : 'unverified', + }, + }); + } else { + await callbacks.onGraphPublished?.(); + } + } return result; } finally { // Readers opened after analyze must not inherit its smaller write budget. setBufferPoolSizeHint(undefined); - await removeColdParseRebuildDir(parseRun.storagePath, true); + if (options.graphPublication && !retainParseStage) { + // A close/publication failure can occur after the durable handoff was + // written but before an AnalyzeResult is returned. Preserve its cache + // even when the receipt itself is unreadable. + retainParseStage = await retainsPreparedGraph(writeTarget.metaDir); + } + if (!retainParseStage) await removeColdParseRebuildDir(parseRun.storagePath, true); discardScopedEmbeddingSpills(); lock.release(); } @@ -1588,7 +2003,7 @@ async function matchesIndexedDirtyWorktree(repoPath: string, meta: RepoMeta): Pr !Array.isArray(indexedDirtyPaths) || indexedDirtyPaths.length === 0 || indexedDirtyPaths.some((rel) => typeof rel !== 'string') || - getIndexIncompleteReasons(meta).length > 0 || + getGraphIncompleteReasons(meta).length > 0 || indexCoverage?.maxFileSizeBytes !== getMaxFileSizeBytes() ) { return false; @@ -1647,8 +2062,22 @@ async function runFullAnalysisInner( runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { const ftsDisabledReason = resolveFtsDisableReason(options.skipFts); - const initAnalysisLbug = (dbPath: string) => - ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath); + const initAnalysisLbug = (dbPath: string) => { + if (options.resources) { + // Only an explicit family budget caps the existing page-scaled sizing. + // Automatic reservations preserve the COPY floor on large-page hosts. + const sizedPool = getEffectiveBufferPoolSize(); + const ceiling = options.resources.parse.graphPoolBytes; + return initLbug(dbPath, { + skipFts: !!ftsDisabledReason, + bufferPoolBytes: options.resources.automatic + ? sizedPool + : Math.min(sizedPool || ceiling, ceiling), + threads: options.resources.parse.nativeThreads, + }); + } + return ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath); + }; const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); @@ -1723,6 +2152,17 @@ async function runFullAnalysisInner( }; const ftsProfile = ftsProfileForContentRetention(contentRetention); const ftsIndexes = getFtsIndexes(ftsProfile); + let graphResourcesAdmitted = false; + const admitGraphResources = async (): Promise => { + if (graphResourcesAdmitted) return; + if (options.graphPublication === 'after-exit') await callbacks.onGraphWork?.(); + if (!options.resources) return; + const { drainEmbeddingSidecarForAnalysis } = + await import('./embeddings/embedding-sidecar-client.js'); + await drainEmbeddingSidecarForAnalysis(); + await drainAnalysisNativeResources(); + graphResourcesAdmitted = true; + }; // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open // (e.g. the embeddings-cache open) falls back to the default until the hint is @@ -1751,6 +2191,7 @@ async function runFullAnalysisInner( // first graph open: here for the paths that open it before the up-to-date // check, and below once that check falls through. const ensurePrivateGraph = async (copy = true): Promise => { + await admitGraphResources(); if (!writeTarget.sharedStore || placement.branch) return; if (!(await ensurePrivateSharedGraph(metaDir, log, { copy }))) { collector.add({ @@ -1760,7 +2201,7 @@ async function runFullAnalysisInner( } // Later dirty-flag writes spread the in-memory metadata; keep them from // re-recording the pointer this slot just left. - delete loadedMeta?.graphPath; + if (loadedMeta?.graphPath !== path.basename(lbugPath)) delete loadedMeta?.graphPath; }; const loadedMeta = await loadMeta(metaDir); if (loadedMeta?.incrementalInProgress || options.repairFts) await ensurePrivateGraph(); @@ -1888,6 +2329,7 @@ async function runFullAnalysisInner( 'Run `gitnexus analyze` (full) to rebuild from scratch.', ); } + await admitGraphResources(); await ensureWritableStorage(); // P1 R8: park a poisoned live WAL before opening. Staging never parks — // the live index next to an unpublished staging file must replay its WAL. @@ -2517,7 +2959,13 @@ async function runFullAnalysisInner( // it means an index predating the field, whose width is unknown but was // consistent with the env that wrote it, and which the fingerprint guard // above already rebuilds — that rebuild is where the stamp lands. - if (existingMeta && embeddingDimsMismatch(existingMeta.embeddingDims, EMBEDDING_DIMS)) { + // Published generations keep vectors in their own typed artifact. Changing + // that artifact's width must not invalidate the independently versioned graph. + if ( + existingMeta && + !(options.graphPublication && existingMeta.graphGeneration) && + embeddingDimsMismatch(existingMeta.embeddingDims, EMBEDDING_DIMS) + ) { // Only NAME a recorded width that could be one, for the reason the // fingerprint guard gates its stamp on `isSchemaFingerprintShaped`: // meta.json is a schema-less JSON.parse of on-disk state, so a value that @@ -2669,7 +3117,10 @@ async function runFullAnalysisInner( JSON.stringify(getIncludeDotDirs()); if ( existingMeta && - !existingMeta.embeddingCheckpoint && + (!existingMeta.embeddingCheckpoint || + (options.graphPublication && + resumeEmbeddingCheckpoint && + !existingMeta.embeddingCheckpoint.recovery)) && !options.force && existingMeta.lastCommit === currentCommit && !ftsModeChanged && @@ -2887,6 +3338,7 @@ async function runFullAnalysisInner( } } + await admitGraphResources(); await ensureWritableStorage(); // A forced rebuild reads the old graph only to carry embeddings over; with // none to carry, copying the shared graph would be thrown away unread. @@ -2894,7 +3346,7 @@ async function runFullAnalysisInner( resumeEmbeddingCheckpoint || _deriveEmbeddingMode(options, existingMeta?.stats?.embeddings ?? 0).shouldLoadCache; await ensurePrivateGraph(!options.force || forcedRebuildReadsOldGraph); - delete existingMeta?.graphPath; + if (existingMeta?.graphPath !== path.basename(lbugPath)) delete existingMeta?.graphPath; // Checkpoint 1, final read: a graph copy that just failed forces the rebuild // before the embedding plan, the parse cache, and the emit mode are decided. applyCollectedForce(); @@ -2913,8 +3365,8 @@ async function runFullAnalysisInner( // proven in-place path; enable it only to test the Windows swap. const windowsSwapOk = process.platform === 'win32' && - options.pdg !== true && - process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1'; + (options.graphPublication !== undefined || + (options.pdg !== true && process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1')); // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: @@ -2953,7 +3405,16 @@ async function runFullAnalysisInner( shouldLoadCache: derivedShouldLoadCache, } = _deriveEmbeddingMode(options, existingEmbeddingCount); const shouldGenerateEmbeddings = derivedShouldGenerateEmbeddings || resumeEmbeddingCheckpoint; - const shouldLoadCache = derivedShouldLoadCache || resumeEmbeddingCheckpoint; + const shouldLoadCache = + !(options.graphPublication && (existingMeta?.vectorGeneration || existingMeta?.embeddingJob)) && + (derivedShouldLoadCache || resumeEmbeddingCheckpoint); + + const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint); + const stagedCheckpoint = existingMeta?.embeddingCheckpoint; + const inheritedUnsafeNodeIds = new Set([ + ...pendingEmbeddingNodeIds, + ...(stagedRecovery?.unsafeNodeIds ?? []), + ]); if (options.dropEmbeddings && existingEmbeddingCount > 0) { log( @@ -2989,6 +3450,42 @@ async function runFullAnalysisInner( // possible the dirty block above already threw a LbugWipeError, so this // open is replay-free by construction (FIX 1 of this shipping review). if (shouldLoadCache && existingMeta) { + if ( + options.graphPublication && + !options.dropEmbeddings && + !existingMeta.graphGeneration && + !existingMeta.legacyVectorSource && + // A supervisor handoff may leave only placeholder metadata on a first + // run. Missing storage is an error only when it claims paid vectors. + (existingEmbeddingCount > 0 || existsSync(lbugPath)) + ) { + const { exportLegacyVectorsFromFile } = await import('./lbug/legacy-vector-export.js'); + const historical = existingMeta.embeddingCheckpoint; + const legacy = await exportLegacyVectorsFromFile(lbugPath, { + metadataDirectory: metaDir, + provenance: { + schemaFingerprint: existingMeta.schemaFingerprint ?? SCHEMA_FINGERPRINT, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: inheritedUnsafeNodeIds, + sourcePoolBytes: options.resources!.embedding.graphPoolBytes, + bufferPoolBytes: options.resources!.embedding.vectorPoolBytes, + threads: options.resources!.embedding.nativeThreads, + signal: callbacks.signal, + }); + if (legacy) { + await saveMeta(metaDir, { ...existingMeta, legacyVectorSource: legacy }); + existingMeta.legacyVectorSource = legacy; + } + } // Cache initialization is writable and may discard quarantined WALs. // Preserve the live family before this first open, including runs whose // incremental write later escalates to a staged full rebuild. Keep this @@ -3021,12 +3518,6 @@ async function runFullAnalysisInner( // A checkpoint's pending decision and its paid, complete vectors are // independent: --force discards the former but can still reuse the latter. // Select only the explicitly referenced generation, never an orphan by age. - const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint); - const stagedCheckpoint = existingMeta?.embeddingCheckpoint; - const inheritedUnsafeNodeIds = new Set([ - ...pendingEmbeddingNodeIds, - ...(stagedRecovery?.unsafeNodeIds ?? []), - ]); if (shouldLoadCache && !options.dropEmbeddings && stagedRecovery && stagedCheckpoint) { if (!embeddingIdentityForRun) { const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js'); @@ -3161,6 +3652,8 @@ async function runFullAnalysisInner( inputs.map((file) => file.path), ); } + callbacks.signal?.throwIfAborted(); + const { runPipelineFromRepo } = await import('./ingestion/pipeline.js'); pipelineResult = await runPipelineFromRepo( repoPath, (p) => { @@ -3172,10 +3665,12 @@ async function runFullAnalysisInner( progress(p.phase, scaled, message); }, { + signal: callbacks.signal, parseCache, parsedFileStorePath: coldParseRebuildDir, parseWorkerUrl: parseIdentityResolution.ok ? parseIdentityResolution.workerUrl : undefined, workerPoolSize: options.workerPoolSize, + ...(options.resources ? { workerResources: options.resources.parse } : {}), maxProcesses: processDetectionBudget.maxProcesses, maxProcessBranching: processDetectionBudget.overridden.maxProcessBranching ? processDetectionBudget.maxProcessBranching @@ -3241,6 +3736,7 @@ async function runFullAnalysisInner( } // ── Phase 2: LadybugDB (60–85%) ────────────────────────────────── + callbacks.signal?.throwIfAborted(); progress('lbug', 60, 'Loading into LadybugDB...'); // Parsing and graph construction always see the original source. Apply the @@ -3415,6 +3911,33 @@ async function runFullAnalysisInner( // orphan sweep (sweepStagingArtifacts) reclaims stragglers on the next // acquire. The `.staging.` prefix is what that sweep matches. let buildPath = useAtomicSwap ? `${lbugPath}.staging.${randomUUID()}` : lbugPath; + let inPlaceAttempt: string | undefined; + let inPlaceVectorCoverage: RepoMeta['vectorGeneration']; + const saveGraphDirtyState = async (metadata: RepoMeta): Promise => { + if (!options.graphPublication || buildPath !== lbugPath) { + await saveMeta(metaDir, metadata); + return; + } + if (!inPlaceAttempt) { + const dirty = await beginInPlaceAnalysis(metaDir, metadata, (meta) => + saveMeta(metaDir, meta), + ); + inPlaceAttempt = dirty.incrementalInProgress?.attemptId; + inPlaceVectorCoverage = dirty.vectorGeneration; + return; + } + if (!metadata.incrementalInProgress) throw new Error('Missing in-place write state'); + await saveMeta(metaDir, { + ...metadata, + graphGeneration: undefined, + vectorGeneration: inPlaceVectorCoverage, + incrementalInProgress: { + ...metadata.incrementalInProgress, + attemptId: inPlaceAttempt, + writePlan: 'in-place', + }, + }); + }; if (isIncremental && hashDiff) { log( @@ -3431,7 +3954,7 @@ async function runFullAnalysisInner( // force-rebuild a healthy index after a crash before swap. if (!atomicIncremental) { const now = Date.now(); - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta!, incrementalInProgress: { startedAt: now, @@ -3466,9 +3989,9 @@ async function runFullAnalysisInner( // clean-tree run's fast path would certify a destroyed DB (or, after a // pdg flip, certify zombie/missing BasicBlock rows indefinitely). // toWriteCount: 0 is the full-path sentinel (no incremental write set). - if (existingMeta) { + if (existingMeta && (!options.graphPublication || !useAtomicSwap)) { const now = Date.now(); - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta, incrementalInProgress: { startedAt: now, @@ -3640,6 +4163,7 @@ async function runFullAnalysisInner( // Resolution groups can change edges between unchanged files without an // import or a new-graph edge touching the changed/deleted member. Expand // before the importer walk so dependents of these siblings refresh too. + const { collectResolutionGroupFiles } = await import('./incremental/resolution-groups.js'); for (const filePath of collectResolutionGroupFiles( [...hashDiff.toWrite, ...hashDiff.deleted], allFilePaths, @@ -3671,7 +4195,7 @@ async function runFullAnalysisInner( ): Promise => { // Do not stamp live metadata while writing a staging database. if (buildPath !== lbugPath) return; - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...existingMeta!, incrementalInProgress: { startedAt: dirtyStartedAt, @@ -4439,7 +4963,7 @@ async function runFullAnalysisInner( // shipped defaults is a match, so a budget-mismatch derived rewrite that // dies in FTS would otherwise recertify the rewritten Community/Process // rows on a flagless retry. - await saveMeta(metaDir, { + await saveGraphDirtyState({ ...base, incrementalInProgress: buildFtsDirtyStamp({ prior: base.incrementalInProgress, @@ -4455,6 +4979,7 @@ async function runFullAnalysisInner( } // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── + callbacks.signal?.throwIfAborted(); // The analyze (write) path owns building the search indexes, so it uses // the `auto` install policy (LOAD-first, then one bounded INSTALL) — // symmetric with the VECTOR/embeddings path below and consistent with the @@ -4968,7 +5493,7 @@ async function runFullAnalysisInner( } let stagedCheckpointEmbeddingCount: number | undefined; - if (!embeddingSkipped) { + if (!embeddingSkipped && !options.graphPublication) { const { isHttpMode } = await import('./embeddings/http-client.js'); const httpMode = isHttpMode(); progress( @@ -5191,6 +5716,7 @@ async function runFullAnalysisInner( } // ── Phase 5: Finalize (98–100%) ─────────────────────────────────── + callbacks.signal?.throwIfAborted(); progress('done', 98, 'Saving metadata...'); // Count embeddings in the index (cached + newly generated). Tri-state, and @@ -5306,10 +5832,13 @@ async function runFullAnalysisInner( // final meta, written after the swap; never publish it at a checkpoint. const latestMetaForCount = embeddingCount === undefined ? ((await loadMeta(metaDir)) ?? existingMeta) : undefined; - const persistedEmbeddingCount = resolvePersistedEmbeddingCount( - measuredEmbeddingCount, - stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, - ); + const persistedEmbeddingCount = + options.graphPublication && existingMeta?.vectorGeneration && !options.dropEmbeddings + ? existingMeta.vectorGeneration.chunks + : resolvePersistedEmbeddingCount( + measuredEmbeddingCount, + stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, + ); const { getRuntimeCapabilities } = await import('./platform/capabilities.js'); const runtimeCapabilities = getRuntimeCapabilities(); @@ -5354,9 +5883,18 @@ async function runFullAnalysisInner( // RepoMeta's status unions (tri-review 4669518496 P1/U3) — an unannotated // literal widens the vectorSearch.status ternary to `string` and the // honesty contract silently decays to "whatever interpolates". - const meta: RepoMeta = { + let meta: RepoMeta = { repoPath, storagePath, + ...(options.graphPublication && !options.dropEmbeddings + ? { + graphGeneration: existingMeta?.graphGeneration, + vectorGeneration: existingMeta?.vectorGeneration, + embeddingJob: existingMeta?.embeddingJob, + legacyVectorSource: existingMeta?.legacyVectorSource, + } + : {}), + ...(path.basename(lbugPath) !== LBUG_DIRECTORY ? { graphPath: path.basename(lbugPath) } : {}), lastCommit: currentCommit, indexedAt: new Date().toISOString(), contentRetention, @@ -5475,7 +6013,8 @@ async function runFullAnalysisInner( // dirty flag (full and incremental success paths converge here). // Derived digest of the DDL this run created the tables from (#2798). // Git-only: non-git repos never take the incremental path. - schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, + schemaFingerprint: + options.graphPublication || hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), // Resolution runs again on warm parse-cache hits. Replace the receipt, // including complete-empty, so corrected/deleted source clears old gaps. @@ -5537,7 +6076,12 @@ async function runFullAnalysisInner( incrementalInProgress: undefined as RepoMeta['incrementalInProgress'], // Cleared on a clean run; otherwise the marker Phase 4/5 minted above // (see the `kind` doc in repo-manager.ts). - embeddingCheckpoint: pendingEmbeddingCheckpoint, + embeddingCheckpoint: + options.graphPublication && !options.dropEmbeddings + ? resumeEmbeddingCheckpoint + ? existingMeta?.embeddingCheckpoint + : undefined + : pendingEmbeddingCheckpoint, // The effective pdg config this run's DB rows were built under // (#2099 F1). `undefined` on pdg-off runs — this meta is a fresh // literal (no spread of existingMeta), so omission is what CLEARS the @@ -5588,11 +6132,88 @@ async function runFullAnalysisInner( // Stop the manual checkpoint driver before closeLbug so its // in-flight CHECKPOINT cannot race the `safeClose` CHECKPOINT. await walCheckpointDriver.stop(); - if (useAtomicSwap && !wroteChangedSubgraphOnly && !(await runCheckpointWithRetry()).flushed) { + if ( + (useAtomicSwap || options.graphPublication) && + !wroteChangedSubgraphOnly && + !(await runCheckpointWithRetry()).flushed + ) { // Publication requires durability even when periodic/manual checkpoints // are disabled. A best-effort close cannot certify the staged file. throw new Error('Cannot publish staged index: final checkpoint could not be verified.'); } + let preparedGraph: PreparedAnalysisHandoff | undefined; + if (options.graphPublication) { + if (!meta.parseCacheIdentity) + throw new Error('Graph publication requires verified parser provenance'); + if (!existingMeta?.graphGeneration && !options.dropEmbeddings && !meta.legacyVectorSource) { + const { exportLegacyVectors } = await import('./lbug/legacy-vector-export.js'); + const historical = existingMeta?.embeddingCheckpoint; + meta.legacyVectorSource = await exportLegacyVectors({ + metadataDirectory: metaDir, + query: executeQuery, + provenance: { + schemaFingerprint: existingMeta?.schemaFingerprint ?? SCHEMA_FINGERPRINT, + ...(historical + ? { + recordedIdentity: { + provider: historical.provider, + model: historical.model, + dimensions: historical.dimensions, + }, + } + : {}), + }, + excluded: new Set([ + ...pendingEmbeddingNodeIds, + ...inheritedUnsafeNodeIds, + ...restoreFailedNodeIds, + ]), + bufferPoolBytes: options.resources!.embedding.vectorPoolBytes, + threads: options.resources!.embedding.nativeThreads, + }); + } + if (meta.embeddingCheckpoint?.recovery) { + // The complete staged groups now live in the independent legacy source. + // Publish the remaining retry decision without the retired staging + // pointer; the embedding continuation consumes the durable source. + meta.embeddingCheckpoint = { ...meta.embeddingCheckpoint }; + delete meta.embeddingCheckpoint.recovery; + } + const content = createHash('sha256'); + for (const entry of [...newFileHashes].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) + content.update(JSON.stringify(entry)).update('\0'); + const receipt = await prepareAnalysisPublication( + metaDir, + buildPath, + meta, + content.digest('hex'), + useAtomicSwap + ? { waitForProcessExit: options.graphPublication === 'after-exit' } + : { + mode: 'in-place', + attemptId: inPlaceAttempt!, + waitForProcessExit: options.graphPublication === 'after-exit', + }, + ); + if (parseCache) { + try { + await stageAnalysisParseCache(metaDir, receipt.id, parseCache); + } catch (error) { + log(`Warning: could not stage parser cache (${(error as Error).message}); continuing.`); + } + } + if (options.graphPublication === 'after-exit') { + preparedGraph = { + version: 1, + publicationId: receipt.id, + repoPath, + storagePath, + metadataDirectory: metaDir, + repoName: await preparedAnalysisName(repoPath, options), + commit: currentCommit, + }; + } + } // CLI callers (about to process.exit) skip the native close to dodge a // LadybugDB destructor double-free after --pdg writes — closeLbugBeforeExit // CHECKPOINTs for durability then leaves the handles for process exit to @@ -5602,7 +6223,8 @@ async function runFullAnalysisInner( // same-process open file can't be renamed), so it forces a real close — // safe because windowsSwapOk excludes --pdg (the #2264 case). POSIX renames // an open file, so it keeps the skip-native-close there. - const forceRealCloseForSwap = useAtomicSwap && process.platform === 'win32'; + const forceRealCloseForSwap = + useAtomicSwap && process.platform === 'win32' && !options.graphPublication; await (options.skipNativeCloseOnExit && !forceRealCloseForSwap ? closeLbugBeforeExit() : closeLbug()); @@ -5626,7 +6248,7 @@ async function runFullAnalysisInner( () => false, ) : false; - if (useAtomicSwap && builtDbExists) { + if (!options.graphPublication && useAtomicSwap && builtDbExists) { await assertLbugPublicationReady(buildPath); const releaseInitLock = await acquireInitLock(lbugPath); try { @@ -5644,8 +6266,22 @@ async function runFullAnalysisInner( // than latching on the old one. Leaving the dirty flag set across the swap // is a crash-safety improvement: a failed swap leaves the previous index // live and the next run recovers via the full-rebuild path. - await saveMeta(metaDir, meta); - sweepStagingArtifacts(metaDir, log); + if (options.graphPublication === 'inline') { + const { verifyPublishedAnalysisGraph } = await import('./analysis-graph-verification.js'); + const published = await publishPreparedAnalysis(metaDir, { + saveMetadata: (metadata) => saveMeta(metaDir, metadata), + verify: (file, metadata) => + verifyPublishedAnalysisGraph(file, metadata, { + bufferPoolBytes: options.resources!.parse.graphPoolBytes, + threads: options.resources!.parse.nativeThreads, + }), + }); + if (!published) throw new Error('Prepared graph disappeared before publication'); + meta = published; + } else if (!preparedGraph) { + await saveMeta(metaDir, meta); + } + if (!preparedGraph) sweepStagingArtifacts(metaDir, log); // Registry freshness is published only after the graph and its metadata. // A failed close, swap, or metadata save must leave the previous registry @@ -5659,17 +6295,19 @@ async function runFullAnalysisInner( // (after applying the precedence chain in registerRepo) — reuse it // so AGENTS.md / skill files reference the same name MCP clients // will look up (#979). - const projectName = await registerRepo(repoPath, meta, { - name: options.registryName, - onRename: (previousName, nextName) => - log(`Registry name changed: "${previousName}" -> "${nextName}".`), - allowDuplicateName: options.allowDuplicateName, - // Non-primary branch runs upsert into the entry's branches[]; the - // primary/flat run (placement.branch === undefined) refreshes the - // top-level fields (#2106). - branch: placement.branch, - storagePath, - }); + const projectName = + preparedGraph?.repoName ?? + (await registerRepo(repoPath, meta, { + name: options.registryName, + onRename: (previousName, nextName) => + log(`Registry name changed: "${previousName}" -> "${nextName}".`), + allowDuplicateName: options.allowDuplicateName, + // Non-primary branch runs upsert into the entry's branches[]; the + // primary/flat run (placement.branch === undefined) refreshes the + // top-level fields (#2106). + branch: placement.branch, + storagePath, + })); // ── #2354: the flat workspace slot has adopted this run's branch ────── // Drop a now-shadowed `branches//` sub-index for the same label @@ -5681,7 +6319,7 @@ async function runFullAnalysisInner( // what applyBranchScope trusts. Retried by the next content-changing run // (same-commit fast-path runs skip it: their guard compares the // already-stamped meta label). - if (!placement.branch && branchLabel) { + if (!preparedGraph && !placement.branch && branchLabel) { try { await adoptFlatBranchLabel(repoPath, branchLabel, storagePath); } catch (e) { @@ -5743,96 +6381,29 @@ async function runFullAnalysisInner( // so the cache file size stays bounded across runs (chunks whose // composition no longer matches anything in the current scan are dead // weight; the parse phase populates `usedKeys` as it processes chunks). - const saveCaches = async (): Promise => { - if (!parseCache) return; - // #2106 R6: the parse cache + durable store are shared across branches. - // Before pruning to this run's keys, fold in the OTHER branches' recorded - // chunk keys so a branch switch doesn't evict their still-live shards. - // Adding to usedKeys makes them survive pruneCache AND land in the saved - // index (saveParseCache builds the index from usedKeys). Excludes this - // run's own meta dir, so a single-branch repo folds in nothing → prune - // set byte-identical to today. A shared store (#3352) folds in every - // member checkout and commit graph the same way. - // An unlistable store directory starts the fold incomplete, so the - // retention branch below keeps other slots' chunks. - const listing = writeTarget.sharedStore - ? await listStoreMetaRoots(writeTarget.sharedStore) - : { roots: [storagePath], complete: true }; - const siblingKeys = new Set(); - let complete = listing.complete; - for (const root of listing.roots) { - const folded = await collectBranchCacheKeys(root, metaDir); - for (const k of folded.keys) siblingKeys.add(k); - if (!folded.complete) complete = false; - } - // Another slot may have published since this run loaded its cache. - // Refresh eligibility inside the publication lock for EVERY run. This - // is also the first persistent read for a bypass run, after parsing. - // Metadata alone cannot admit foreign shards, and live entries cannot - // replace this run's staged repairs or revive its retired chunks. - const retained = await loadParseCache(cacheRoot, parseCache.identity, coldParseRebuildDir); - parseCache.storagePath = cacheRoot; - parseCache.onDiskKeys = new Set(); - for (const key of retained.onDiskKeys ?? []) { - if ( - !parseCache.stagedKeys?.has(key) && - !parseCache.staleKeys?.has(key) && - (parseCache.usedKeys.has(key) || !complete || siblingKeys.has(key)) - ) { - parseCache.onDiskKeys.add(key); - } - } - for (const [key, value] of retained.entries) { - if ( - !parseCache.usedKeys.has(key) && - !parseCache.stagedKeys?.has(key) && - !parseCache.staleKeys?.has(key) && - (!complete || siblingKeys.has(key)) - ) { - parseCache.entries.set(key, value); - } - } - if (complete) { - for (const k of siblingKeys) parseCache.usedKeys.add(k); - } else { - // Fail-safe toward retention: a sibling meta was unreadable, so keep - // everything currently loaded rather than evict on incomplete info. - log('Parse cache: a branch meta was unreadable — retaining all cached chunks (#2106).'); - for (const k of parseCache.entries.keys()) parseCache.usedKeys.add(k); - for (const k of parseCache.onDiskKeys ?? []) parseCache.usedKeys.add(k); - } - const pruned = pruneCache(parseCache, parseCache.usedKeys); - if (pruned > 0) { - log(`Parse cache: pruned ${pruned} stale chunk entries`); - } - const savedKeys = await saveParseCache(cacheRoot, parseCache); - // Prune the durable ParsedFile store to EXACTLY the parse cache's - // surviving keys (#2038 warm-cache coverage), so the two content-addressed - // stores stay coherent: a chunk is "cached" iff both its parse-cache shard - // and its durable shards exist. A retired chunk — worker-quarantined, or - // one whose failed durable reset left an uncleared generation behind - // (#3204) — is filtered out of `savedKeys`, so it drops out of the - // durable index here and re-dispatches next run. Same try/catch — a - // durable-store write must never - // break an otherwise successful run (next run treats it as a miss). - await mergeStagedDurableParsedFileStore( - cacheRoot, - parseCache.writeStoragePath ?? cacheRoot, - parseCache.identity.base, - new Set(savedKeys), - ); - }; try { - await (writeTarget.sharedStore - ? withStoreLock(writeTarget.sharedStore, 'cache', saveCaches) - : saveCaches()); + if (parseCache && !preparedGraph) { + await publishAnalysisParseCache(parseCache, { + storagePath, + metadataDirectory: metaDir, + cacheRoot, + publishedGeneration: meta.graphGeneration?.id, + sharedStore: writeTarget.sharedStore, + collectBranchKeys: collectBranchCacheKeys, + log, + }); + } } catch (e) { log(`Warning: could not save parse cache (${(e as Error).message}); continuing.`); } - progress('done', 100, 'Done'); + if (preparedGraph) + progress('graph-prepared', 98, 'Graph checkpointed; awaiting process release'); + else if (options.graphPublication) + callbacks.onProgress('graph-ready', 98, 'Graph published', { graphPhase: 'ready' }); + else progress('done', 100, 'Done'); - await removeColdParseRebuildDir(coldParseRebuildDir, true); + if (!preparedGraph) await removeColdParseRebuildDir(coldParseRebuildDir, true); return { repoName: projectName, @@ -5841,6 +6412,7 @@ async function runFullAnalysisInner( stats: meta.stats, rebuildReasons: collector.keys(), pipelineResult, + ...(preparedGraph ? { preparedGraph } : {}), ...(graphWriteCollapsed ? { graphWriteCollapsed } : {}), ftsSkipped: !ftsReady, ftsSkipReason: ftsReady ? undefined : ftsSkipReason, @@ -5895,7 +6467,8 @@ async function runFullAnalysisInner( // differs from `lbugPath` exactly when the atomic-swap plan is in effect, // and the live index is never that path. Best-effort by construction — the // rethrow below is the surface, and the lock's sweep remains the backstop. - if (stagedBuild) { + const retainPrepared = options.graphPublication && (await retainsPreparedGraph(metaDir)); + if (stagedBuild && !retainPrepared) { try { const recovery = resolveEmbeddingRecovery( metaDir, @@ -5908,7 +6481,7 @@ async function runFullAnalysisInner( /* swallow — orphan reclamation must never mask the real failure */ } } - await removeColdParseRebuildDir(coldParseRebuildDir, true); + if (!retainPrepared) await removeColdParseRebuildDir(coldParseRebuildDir, true); discardCachedEmbeddingSpill(); // What the caller sees: `err` itself (identity and prototype preserved, callers // tell IndexLockTimeoutError and other domain failures apart with `instanceof`), diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 5e06a68fa..25d5565f4 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -37,10 +37,13 @@ import { type RegistryEntry, } from '../storage/repo-manager.js'; import { isMissingFilesystemError, loadMeta, type RepoMeta } from '../storage/repo-meta.js'; +import { retainSharedVectorArtifact } from '../storage/shared-vector-artifact.js'; +import { selectVectorGeneration } from '../storage/vector-generation.js'; import { cloneStoreKey, commitGraphDir, resolveGraphPath, + resolveGraphWritePath, resolveSharedStore, sharedStoreLayout, storeRootOfCheckoutSlot, @@ -58,6 +61,7 @@ import { import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { wipeLbugDbFiles } from './lbug/lbug-adapter.js'; import { inspectLbugSidecars } from './lbug/sidecar-recovery.js'; +import { getExactScanLimit } from './platform/capabilities.js'; type Log = (msg: string) => void; @@ -103,11 +107,117 @@ export const featureKeyOf = (meta: RepoMeta): string => { for (const [k, v] of Object.entries(meta)) { if (!FEATURE_KEY_EXCLUDED.has(k)) kept[k] = v; } - // Embedding presence changes graph content but lives only in `stats`. - kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0; + if (hasSeparateGraphGeneration(meta)) { + // Logical IDs and separate vector work do not change the graph's facts. + // Keep all unknown generation/capability fields in the denylist hash. + const generation: Record = { ...meta.graphGeneration }; + delete generation.id; + kept.graphGeneration = generation; + for (const key of ['embeddingJob', 'vectorGeneration', 'legacyVectorSource', 'embeddingDims']) + delete kept[key]; + if (meta.capabilities) { + const capabilities: Record = { ...meta.capabilities }; + delete capabilities.vectorSearch; + kept.capabilities = capabilities; + } + } else { + // Legacy embeddings still live inside the graph database. + kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0; + } return createHash('sha256').update(stableStringify(kept)).digest('hex').slice(0, 16); }; +function hasSeparateGraphGeneration(meta: RepoMeta): boolean { + const generation = meta.graphGeneration; + return ( + generation?.version === 1 && + /^[a-f0-9]{32}$/.test(generation.id) && + /^[a-f0-9]{64}$/.test(generation.coveredContent) + ); +} + +/** Share completed immutable files only. The original slot keeps its mutable + * job and receipts; siblings never inherit ownership of that paid work. */ +async function retainSharedMetadata( + source: string, + destination: string, + meta: RepoMeta, + allowCopy = false, +): Promise { + if ( + !meta.graphGeneration && + !meta.embeddingJob && + !meta.vectorGeneration && + !meta.legacyVectorSource + ) + return { ...meta }; + if (!hasSeparateGraphGeneration(meta)) + throw new Error('Cannot share an unverified logical graph generation'); + const exported = { ...meta }; + delete exported.embeddingJob; + delete exported.embeddingCheckpoint; + delete exported.legacyVectorSource; + const selected = selectVectorGeneration( + source, + exported, + resolveGraphPath(source), + Number.MAX_SAFE_INTEGER, + ); + const vector = meta.vectorGeneration; + const active = meta.embeddingJob?.artifactIdentity ?? meta.embeddingJob?.identity; + const completed = + vector && + active !== (vector.artifactIdentity ?? vector.identity) && + (selected.kind === 'ready' || + (selected.kind === 'unavailable' && selected.reason === 'embeddings-empty')); + if (completed) { + await retainSharedVectorArtifact(source, destination, vector, allowCopy); + } else { + delete exported.vectorGeneration; + delete exported.embeddingDims; + } + exported.stats = { ...meta.stats, embeddings: completed ? vector.chunks : 0 }; + if (meta.capabilities) { + const exactScanLimit = meta.capabilities.vectorSearch?.exactScanLimit ?? getExactScanLimit(); + const readable = + completed && + vector.chunks > 0 && + (vector.indexKind === 'hnsw' || vector.chunks <= exactScanLimit); + exported.capabilities = { + ...meta.capabilities, + vectorSearch: { + provider: completed && vector.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: readable + ? vector.indexKind === 'hnsw' + ? 'vector-index' + : 'exact-scan' + : 'unavailable', + exactScanLimit, + coverage: completed + ? vector.graphGeneration === meta.graphGeneration?.id + ? 'current' + : 'preserved' + : 'unknown', + ...(!readable + ? { + reason: + completed && vector.chunks ? 'vector-index-pending' : 'embeddings-unavailable', + } + : {}), + ...(completed ? { sourceGraphGeneration: vector.graphGeneration } : {}), + }, + }; + } + return exported; +} + +/** Commit metadata intentionally omits checkout-only required RepoMeta fields. */ +function commitMetadata(meta: RepoMeta): RepoMeta { + const result: Record = { ...meta }; + for (const field of CHECKOUT_FIELDS) delete result[field]; + return result as unknown as RepoMeta; +} + const exists = (p: string): Promise => fs.access(p).then( () => true, @@ -204,7 +314,7 @@ const seedFromLocalIndex = async ( source: string, log: Log, ): Promise => { - const sourceGraph = path.join(source, LBUG_DIRECTORY); + let sourceGraph = resolveGraphPath(source); const usable = (m: RepoMeta | null): m is RepoMeta => !!m && !m.incrementalInProgress && @@ -219,13 +329,22 @@ const seedFromLocalIndex = async ( } let meta: RepoMeta; try { - if (lock.lockFree || (await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false; + if (lock.lockFree) return false; // Re-read under the lock: an analyze that finished while this waited // rewrote both, and the copied graph must match the saved metadata. const locked = await loadMeta(source); if (!usable(locked)) return false; - meta = locked; + sourceGraph = resolveGraphPath(source); + if ((await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false; + if ( + locked.embeddingJob && + registryPathEquals(canonicalizePath(locked.repoPath), canonicalizePath(repoPath)) + ) + throw new Error( + "Finish this checkout's pending embedding job in its original storage before shared-store adoption", + ); await fs.mkdir(slot, { recursive: true }); + meta = await retainSharedMetadata(source, slot, locked, true); try { await cloneGraphFile(sourceGraph, path.join(slot, LBUG_DIRECTORY)); } catch (err) { @@ -279,13 +398,18 @@ export const seedSharedSlot = async ( (await withStoreLock(layout, 'publish', async () => { const graph = path.join(seed.dir, LBUG_DIRECTORY); if (!(await exists(graph))) return false; + // Vector publication can advance this snapshot after pickSeed. Re-read + // while the publish lock also prevents its source from being collected. + const current = await loadMeta(seed.dir); + if (!current) return false; await fs.mkdir(layout.checkoutSlot, { recursive: true }); + const retained = await retainSharedMetadata(seed.dir, layout.checkoutSlot, current); const meta: RepoMeta = { - ...seed.meta, + ...retained, repoPath, storagePath: layout.checkoutSlot, graphPath: graph, - lastCommit: upToDate ? seed.meta.lastCommit : '', + lastCommit: upToDate && canReuseGraph(current) ? current.lastCommit : '', }; delete meta.incrementalInProgress; await saveMeta(layout.checkoutSlot, meta); @@ -316,7 +440,7 @@ export const ensurePrivateSharedGraph = async ( log: Log, opts: { copy?: boolean } = {}, ): Promise => { - const own = path.join(slot, LBUG_DIRECTORY); + const own = resolveGraphWritePath(slot); const pointed = resolveGraphPath(slot); if (pointed === own) return true; const meta = await loadMeta(slot); @@ -384,7 +508,7 @@ export const publishSharedGraph = async ( const slot = layout.checkoutSlot; const meta = await loadMeta(slot); if (!meta) return; - const own = path.join(slot, LBUG_DIRECTORY); + const own = resolveGraphWritePath(slot); // A graph built while files were dirty must not become shared until analyze // has reconciled those paths with disk and cleared them from the coverage receipt. @@ -397,10 +521,14 @@ export const publishSharedGraph = async ( // FTS-only repair preserves the graph's existing parser provenance. meta.parseCacheIdentity !== undefined && !meta.incrementalInProgress && + !meta.graphWriteCollapsed && builtClean && // Embeddings still owed stay with this checkout, which finishes them; a // commit graph never changes, so a shared copy would stay short for good. !meta.embeddingCheckpoint && + // A migration receipt owns historical vectors in this private slot. Do + // not replace its graph or strip that evidence before migration finishes. + !Object.hasOwn(meta, 'legacyVectorSource') && // A sparse or partial checkout builds a graph missing the files it hides. // Every sparse mode marks those entries skip-worktree, which this rejects // (git-utils.test.ts covers no-cone, cone and sparse-index checkouts). @@ -421,8 +549,9 @@ export const publishSharedGraph = async ( const ownIsBetter = published && (!targetMeta || - !!targetMeta.embeddingCheckpoint || - (targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)) && + (!hasSeparateGraphGeneration(meta) && + (!!targetMeta.embeddingCheckpoint || + (targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)))) && (await exists(own)); if (ownIsBetter) { published = false; @@ -431,22 +560,25 @@ export const publishSharedGraph = async ( ); } else if (published) { try { + if (hasSeparateGraphGeneration(meta) && meta.vectorGeneration && !meta.embeddingJob) { + const retained = await retainSharedMetadata(slot, target, meta); + if (retained.vectorGeneration) await saveMeta(target, commitMetadata(retained)); + } await wipeLbugDbFiles(own); } catch (err) { // An open reader (Windows) can block the delete. The analysis // already succeeded; keep the private graph and try next run. published = false; log( - `Shared store: could not drop the private graph (${(err as Error).message}); keeping it.`, + `Shared store: could not finalize the shared graph (${(err as Error).message}); keeping the existing graph.`, ); } } else if ((await exists(own)) && (await inspectLbugSidecars(own)).kind === 'clean') { await fs.mkdir(layout.commitsDir, { recursive: true }); const staging = path.join(layout.commitsDir, `.publish-${randomUUID()}`); await fs.mkdir(staging); - const commitMeta: Record = { ...meta }; - for (const field of CHECKOUT_FIELDS) delete commitMeta[field]; try { + const commitMeta = commitMetadata(await retainSharedMetadata(slot, staging, meta)); await fs.rename(own, path.join(staging, LBUG_DIRECTORY)); await fs.writeFile(path.join(staging, INDEX_METADATA_FILE), JSON.stringify(commitMeta)); await fs.rename(staging, target); @@ -481,7 +613,11 @@ export const publishSharedGraph = async ( meta.graphPath = targetGraph; await saveMeta(slot, meta); } - } else if (meta.graphPath !== undefined && (await exists(own))) { + } else if ( + meta.graphPath !== undefined && + path.isAbsolute(meta.graphPath) && + (await exists(own)) + ) { delete meta.graphPath; await saveMeta(slot, meta); } diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 54c7e0e1b..667254258 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -116,6 +116,8 @@ import { embeddingDimsMismatch, } from '../../core/lbug/schema.js'; import { getExactScanLimit } from '../../core/platform/capabilities.js'; +import { metadataMatchesGraph, selectVectorGeneration } from '../../storage/vector-generation.js'; +import { pinRepo } from '../../core/lbug/pool-adapter.js'; import { PhaseTimer } from '../../core/search/phase-timer.js'; import { ftsDegradedWarning, ftsQueryFailedWarning } from '../../core/search/fts-indexes.js'; import { getFtsDisabledReason, type FtsDisabledReason } from '../../core/search/fts-policy.js'; @@ -1850,6 +1852,17 @@ export class LocalBackend { if (!pending) { pending = loadOptionalMeta(path.dirname(repo.lbugPath)).then((meta) => { if (!meta) return null; + if ( + (meta.graphPath !== undefined || meta.graphGeneration) && + !metadataMatchesGraph(path.dirname(repo.lbugPath), meta, repo.lbugPath) + ) { + // A newer immutable path was published after this request resolved + // its handle. Keep the old reader's metadata, never mix generations. + const previous = this.lastObservedPoolState.get(repo.lbugPath); + if (previous?.metadata) return previous.metadata; + repo.generationUnknown = true; + return null; + } // Readers never use ingestion manifests. Do not keep their per-file // hashes/cache keys or pending embedding IDs in an opened snapshot. return { @@ -3562,6 +3575,7 @@ export class LocalBackend { chain_depth?: number; }, ): Promise { + const release = pinRepo(repo.lbugPath); try { const result = await this._queryImpl(repo, params); if (!result.error) assertResponseIdentities(result); @@ -3571,6 +3585,8 @@ export class LocalBackend { return { error: err.message, recoverySuggestion: SYMBOL_IDENTITY_RECOVERY_SUGGESTION }; } throw err; + } finally { + release(); } } @@ -4575,55 +4591,103 @@ export class LocalBackend { // call's width, and the catch below must only clear an entry it did not set. let embeddedDims: number | undefined; try { - // Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off) - // determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only - // the count is read, to decide whether to load the embedding model at all. - const tableCheck = await executeQuery( - repo.lbugPath, - `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`, - ); - if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) { + if (repo.generationUnknown) { + if (degraded) + degraded.reason = + 'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.'; this.lastQueryEmbeddingDims.delete(repo.lbugPath); - this.logKeywordOnlyEmbeddings(); return []; } - - const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); - const queryVec = await embedQuery(query); - const dims = getEmbeddingDims(); - // #2798: the width this query vector really was produced at — the same - // value the CAST below binds against the index's `FLOAT[N]` column, and - // therefore the only honest comparand for query()'s drift warning. - embeddedDims = dims; - this.lastQueryEmbeddingDims.set(repo.lbugPath, dims); - const queryVecStr = `[${queryVec.join(',')}]`; - const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE); - let vectorReady = false; - try { - vectorReady = await ensureVectorExtension(repo.lbugPath); - } catch (err) { - rethrowSymbolIdentityError(err); - if (!this.warnedVectorLoadFailed) { - this.warnedVectorLoadFailed = true; - logger.warn( - { err }, - 'GitNexus [query:vector]: vector extension load failed; using exact scan fallback', - ); - } + const metadata = await this.readToolMetadata(repo); + if (repo.generationUnknown) { + if (degraded) + degraded.reason = + 'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.'; + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + return []; + } + const selection = selectVectorGeneration( + path.dirname(repo.lbugPath), + metadata, + repo.lbugPath, + getExactScanLimit(), + ); + if (selection.kind === 'unavailable') { + if (degraded && selection.reason === 'embedding-pending') + degraded.reason = + 'Semantic search is unavailable while embeddings are pending. Run gitnexus analyze to resume; graph and keyword search remain available.'; + else if (degraded && selection.reason !== 'embeddings-unavailable') + degraded.reason = `Semantic search is unavailable (${selection.reason}). Run gitnexus analyze; graph and keyword search remain available.`; + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + return []; } - let bestChunks = new Map< string, { distance: number; chunkIndex: number; startLine: number; endLine: number } >(); - // Try the vector lane only after its lazy load succeeds. An unavailable - // extension is already reported by ExtensionManager; an index/query - // failure below gets its own once-per-backend diagnostic before the exact - // scan fallback. - if (vectorReady) { + if (selection.kind === 'ready') { + const { searchVectorGeneration } = + await import('../../core/embeddings/vector-generation-search.js'); + const chunks = await searchVectorGeneration( + { metadataDirectory: path.dirname(repo.lbugPath), metadata, graphPath: repo.lbugPath }, + async () => { + const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); + const queryVec = await embedQuery(query); + embeddedDims = getEmbeddingDims(); + this.lastQueryEmbeddingDims.set(repo.lbugPath, embeddedDims); + return queryVec; + }, + limit, + getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE), + ); + bestChunks = new Map((chunks ?? []).map(({ nodeId, ...chunk }) => [nodeId, chunk])); + if (embeddedDims === undefined) this.lastQueryEmbeddingDims.delete(repo.lbugPath); + } else { + // Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off) + // determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only + // the count is read, to decide whether to load the embedding model at all. + const tableCheck = await executeQuery( + repo.lbugPath, + `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`, + ); + if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) { + this.lastQueryEmbeddingDims.delete(repo.lbugPath); + this.logKeywordOnlyEmbeddings(); + return []; + } + + const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js'); + const queryVec = await embedQuery(query); + const dims = getEmbeddingDims(); + // #2798: the width this query vector really was produced at — the same + // value the CAST below binds against the index's `FLOAT[N]` column, and + // therefore the only honest comparand for query()'s drift warning. + embeddedDims = dims; + this.lastQueryEmbeddingDims.set(repo.lbugPath, dims); + const queryVecStr = `[${queryVec.join(',')}]`; + const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE); + let vectorReady = false; try { - bestChunks = await collectBestChunks(limit, async (fetchLimit) => { - const vectorQuery = ` + vectorReady = await ensureVectorExtension(repo.lbugPath); + } catch (err) { + rethrowSymbolIdentityError(err); + if (!this.warnedVectorLoadFailed) { + this.warnedVectorLoadFailed = true; + logger.warn( + { err }, + 'GitNexus [query:vector]: vector extension load failed; using exact scan fallback', + ); + } + } + + // Try the vector lane only after its lazy load succeeds. An unavailable + // extension is already reported by ExtensionManager; an index/query + // failure below gets its own once-per-backend diagnostic before the exact + // scan fallback. + if (vectorReady) { + try { + bestChunks = await collectBestChunks(limit, async (fetchLimit) => { + const vectorQuery = ` CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit}) YIELD node AS emb, distance @@ -4634,66 +4698,66 @@ export class LocalBackend { ORDER BY distance `; - const embResults = await executeQuery(repo.lbugPath, vectorQuery); - for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0); - return embResults.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - distance: row.distance ?? row[4], - })); - }); - } catch (err) { - rethrowSymbolIdentityError(err); - bestChunks = new Map(); - if (!this.warnedVectorQueryFailed) { - // Rare diagnostic: surface why semantic search fell back to the - // exact scan. Emitted once per `LocalBackend` instance lifetime to - // avoid noisy stderr on hot semantic-search paths (DoD §2.8). - this.warnedVectorQueryFailed = true; - logger.warn( - { err }, - 'GitNexus [query:vector]: vector index query failed; using exact scan fallback', - ); + const embResults = await executeQuery(repo.lbugPath, vectorQuery); + for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); + } catch (err) { + rethrowSymbolIdentityError(err); + bestChunks = new Map(); + if (!this.warnedVectorQueryFailed) { + // Rare diagnostic: surface why semantic search fell back to the + // exact scan. Emitted once per `LocalBackend` instance lifetime to + // avoid noisy stderr on hot semantic-search paths (DoD §2.8). + this.warnedVectorQueryFailed = true; + logger.warn( + { err }, + 'GitNexus [query:vector]: vector index query failed; using exact scan fallback', + ); + } } } - } - if (bestChunks.size === 0) { - const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0); - const exactLimit = getExactScanLimit(); - if (embeddingCount > exactLimit) return []; + if (bestChunks.size === 0) { + const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0); + const exactLimit = getExactScanLimit(); + if (embeddingCount > exactLimit) return []; - const rows = await executeQuery( - repo.lbugPath, - ` + const rows = await executeQuery( + repo.lbugPath, + ` MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding `, - ); - for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); - const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ - nodeId: row.nodeId ?? row[0], - chunkIndex: row.chunkIndex ?? row[1] ?? 0, - startLine: row.startLine ?? row[2] ?? 0, - endLine: row.endLine ?? row[3] ?? 0, - embedding: row.embedding ?? row[4] ?? [], - })); - bestChunks = new Map( - rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [ - row.nodeId, - { - distance: row.distance, - chunkIndex: row.chunkIndex, - startLine: row.startLine, - endLine: row.endLine, - }, - ]), - ); + ); + for (const row of rows) assertQueryIdentity(row, 'nodeId', 0); + const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + embedding: row.embedding ?? row[4] ?? [], + })); + bestChunks = new Map( + rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [ + row.nodeId, + { + distance: row.distance, + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + }, + ]), + ); + } } - if (bestChunks.size === 0) return []; const results: any[] = []; diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index 6a3de9431..49dc3746f 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -9,7 +9,9 @@ import type { LocalBackend } from './local/local-backend.js'; import { checkStaleness } from './staleness.js'; import { loadMeta } from '../storage/repo-manager.js'; import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from '../core/analyzer-identity.js'; -import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { checkoutIsDirectory, contentRetentionFromMeta, @@ -381,6 +383,11 @@ async function getContextResource(backend: LocalBackend, repoName?: string): Pro lines.push(` source_available: ${JSON.stringify(sourceAvailable)}`); lines.push(` runner_identity: ${JSON.stringify(freshMeta?.runnerIdentity ?? null)}`); lines.push(` incomplete_reasons: ${JSON.stringify(incompleteReasons)}`); + lines.push(` graph_incomplete_reasons: ${JSON.stringify(getGraphIncompleteReasons(freshMeta))}`); + const embeddingProgress = await readEmbeddingJobProgress(repo.storagePath, freshMeta); + if (embeddingProgress) lines.push(` embedding_progress: ${JSON.stringify(embeddingProgress)}`); + const capabilities = await readIndexCapabilities(repo.storagePath, freshMeta); + if (capabilities) lines.push(` capabilities: ${JSON.stringify(capabilities)}`); lines.push(` spring_actuator: ${JSON.stringify(freshMeta?.springActuator ?? null)}`); // Surfaced beside the Actuator flag, and it matters more than that one does: // Actuator only annotates nodes the source pass already found, whereas diff --git a/gitnexus/src/server/analyze-job.ts b/gitnexus/src/server/analyze-job.ts index 2ab767e7d..35b073e22 100644 --- a/gitnexus/src/server/analyze-job.ts +++ b/gitnexus/src/server/analyze-job.ts @@ -12,8 +12,9 @@ import { randomUUID } from 'crypto'; import { EventEmitter } from 'events'; import type { ChildProcess } from 'child_process'; +import type { AnalysisPhaseProgress } from '../core/analysis-progress.js'; -export interface AnalyzeJobProgress { +export interface AnalyzeJobProgress extends AnalysisPhaseProgress { phase: string; percent: number; message: string; @@ -88,8 +89,8 @@ export interface AnalyzeJob { const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const JOB_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes -/** How long a cancelled worker gets to exit via IPC before a signal is sent. */ -const CANCEL_GRACE_MS = 15_000; +/** Delay before force-killing an unresponsive standalone child after cancellation. */ +const CANCEL_GRACE_MS = 60_000; export class JobManager { private jobs = new Map(); @@ -208,20 +209,27 @@ export class JobManager { /** Register a child process for a job — enables cancellation and timeout. */ registerChild(jobId: string, child: ChildProcess) { + const previousTimer = this.timeouts.get(jobId); + if (previousTimer) clearTimeout(previousTimer); this.children.set(jobId, child); - // 30-minute timeout - const timer = setTimeout(() => { - const job = this.jobs.get(jobId); - if (job && !this.isTerminal(job.status)) { - this.cancelJob(jobId, 'Analysis timed out (30 minute limit)'); - } - }, JOB_TIMEOUT_MS); + // One deadline covers graph preparation, continuation, and crash retries. + const elapsed = Date.now() - (this.jobs.get(jobId)?.startedAt ?? Date.now()); + const timer = setTimeout( + () => { + const job = this.jobs.get(jobId); + if (job && !this.isTerminal(job.status)) { + this.cancelJob(jobId, 'Analysis timed out (30 minute limit)'); + } + }, + Math.max(0, JOB_TIMEOUT_MS - elapsed), + ); this.timeouts.set(jobId, timer); // Apply a pending cancel BEFORE other `exit` listeners (analyze-launch's // crash-retry) see a still-non-terminal job and fork a replacement worker. const onExit = (): void => { + if (this.children.get(jobId) !== child) return; this.releaseChild(jobId); this.applyPendingCancel(jobId); }; @@ -286,12 +294,8 @@ export class JobManager { /** * Cancel a running job. * - * The worker is asked to stop over IPC first (`{ type: 'cancel' }`), which - * lets it reach a JS-visible safe point and checkpoint before exiting — - * the same cross-platform control path `core/auto-sync` uses. A signal is - * sent only as a bounded fallback: on Windows `child.kill('SIGTERM')` is a - * forceful termination (Node ignores the signal name there), so leading - * with it could kill the worker mid LadybugDB write. + * Request cooperative cancellation over IPC first. A standalone child that + * outlasts the grace period is killed; ownership remains held until its exit. */ cancelJob(jobId: string, reason?: string): boolean { const job = this.jobs.get(jobId); @@ -328,31 +332,25 @@ export class JobManager { } /** - * Ask a worker to shut down: IPC cancel now, signal after a grace period if - * it has not exited on its own. The grace timer is cleared by the child's - * `exit` handler registered in `registerChild`. + * Request cancellation, then stop a wedged standalone process after grace. + * Ownership and the job slot remain held until the actual process exit. */ private requestChildShutdown(jobId: string, child: ChildProcess): void { - let ipcSent = false; if (child.connected) { try { - child.send({ type: 'cancel' }); - ipcSent = true; + child.send({ type: 'cancel' }, () => { + /* Disconnect also requests cancellation. */ + }); } catch { - // Channel already closed — fall through to the signal path. + // The child's disconnect handler cancels without interrupting native work. } } - if (!ipcSent) { - child.kill('SIGTERM'); - return; - } if (this.cancelGraceTimers.has(jobId)) return; const grace = setTimeout(() => { this.cancelGraceTimers.delete(jobId); if (child.exitCode === null && child.signalCode === null) { - // IPC already set the worker's cooperative cancel flag; a second - // SIGTERM is a no-op there. SIGKILL is the actual bounded fallback - // (on Windows `kill()` is already TerminateProcess). + // Process termination preserves crash-recovery receipts. Never use + // this policy for worker_threads running native code in this process. child.kill('SIGKILL'); } }, CANCEL_GRACE_MS); @@ -361,23 +359,14 @@ export class JobManager { } dispose() { - // IPC first so a worker that checks in can stop at a safe point. - // On Windows `child.kill('SIGTERM')` is TerminateProcess — skip that - // immediate kill and leave the 15s grace timer to SIGKILL. On Unix, - // SIGTERM after IPC is cooperative, so the grace timers can be dropped. - const windows = process.platform === 'win32'; + // Cancel jobs between workers too, so an already scheduled retry cannot fork. + for (const job of this.jobs.values()) { + if (!this.hasPendingCancel(job.id)) this.cancelJob(job.id, 'Server shutting down'); + } + // Cooperative cancellation precedes bounded standalone-process shutdown. for (const [jobId, child] of this.children) { - this.requestChildShutdown(jobId, child); - if (!windows) { - child.kill('SIGTERM'); - } + if (!this.hasPendingCancel(jobId)) this.requestChildShutdown(jobId, child); } - this.children.clear(); - if (!windows) { - for (const timer of this.cancelGraceTimers.values()) clearTimeout(timer); - this.cancelGraceTimers.clear(); - } - this.pendingCancelReasons.clear(); for (const controller of this.abortControllers.values()) controller.abort(); this.abortControllers.clear(); @@ -391,6 +380,29 @@ export class JobManager { this.emitter.removeAllListeners(); } + /** Keep shutdown alive until standalone writers have physically exited. */ + async waitForChildren(): Promise { + await Promise.all( + [...this.children.values()].map( + (child) => + new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + const finished = () => { + child.removeListener('exit', finished); + child.removeListener('close', finished); + resolve(); + }; + child.once('exit', finished); + // Spawn failures emit error and close, but never exit. + child.once('close', finished); + }), + ), + ); + } + private isTerminal(status: AnalyzeJob['status']): boolean { return isTerminalJobStatus(status); } @@ -402,7 +414,12 @@ export class JobManager { private cleanup() { const now = Date.now(); for (const [id, job] of this.jobs) { - if (this.isTerminal(job.status) && job.completedAt && now - job.completedAt > JOB_TTL_MS) { + if ( + !this.children.has(id) && + this.isTerminal(job.status) && + job.completedAt && + now - job.completedAt > JOB_TTL_MS + ) { this.jobs.delete(id); this.abortControllers.delete(id); } diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts index 840d18285..a33cf7ff2 100644 --- a/gitnexus/src/server/analyze-launch.ts +++ b/gitnexus/src/server/analyze-launch.ts @@ -17,7 +17,6 @@ import { fork } from 'child_process'; import { fileURLToPath, pathToFileURL } from 'url'; import { createRequire } from 'node:module'; import { INDEX_METADATA_FILE } from '../storage/repo-manager.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { ANALYZE_FORCE_STORAGE_REQUIREMENTS, ANALYZE_STORAGE_REQUIREMENTS, @@ -25,11 +24,19 @@ import { } from '../storage/storage-resolver.js'; import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js'; import { logger } from '../core/logger.js'; +import { + analysisMayEmbed, + analysisSupervisorReservation, + resolveHostAnalysisResources, +} from '../core/analysis-host-resources.js'; +import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; +import type { ContinueMessage, GraphPreparedMessage } from './analyze-worker-protocol.js'; import { loadAnalyzeConfig } from '../cli/analyze-config.js'; import type { RouteHookConfig } from '../core/ingestion/language-provider.js'; -import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; import { isTerminalJobStatus, type JobManager } from './analyze-job.js'; import type { WorkerMessage } from './analyze-worker.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; +import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; const _require = createRequire(import.meta.url); @@ -164,7 +171,7 @@ const waitForSettledIndex = async ( } } - const lbugPath = path.resolve(probeRoot, LBUG_DIRECTORY); + const lbugPath = path.resolve(resolveGraphPath(probeRoot)); const lbugRel = path.relative(storageRoot, lbugPath); if (lbugRel.startsWith('..') || path.isAbsolute(lbugRel)) { return false; @@ -172,8 +179,8 @@ const waitForSettledIndex = async ( const lbugStat = statSync(lbugPath); if (lbugStat.mtimeMs < jobStartMs) return false; - return ['lbug.wal', 'lbug.shadow', 'lbug.wal.checkpoint'].every((name) => { - const sidePath = path.resolve(probeRoot, name); + return ['.wal', '.shadow', '.wal.checkpoint'].every((suffix) => { + const sidePath = path.resolve(`${lbugPath}${suffix}`); const sideRel = path.relative(storageRoot, sidePath); if (sideRel.startsWith('..') || path.isAbsolute(sideRel)) { return false; @@ -223,6 +230,15 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // For waitForSettledIndex: files (re)written by this job have mtimes at or // after this instant. Taken before the fork so no worker write predates it. const jobStartMs = Date.now(); + const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); + const resources = resolveHostAnalysisResources({ + mainHeapMb: + (Number.isInteger(envHeapMb) && envHeapMb > 0 + ? envHeapMb + : Math.min(8192, autoHeapCapMb())) + 48, + includeEmbeddings: await analysisMayEmbed(targetPath, opts), + supervisorBytes: analysisSupervisorReservation(), + }); const analyzeLockKey = await requireStoragePath( targetPath, opts.force ? ANALYZE_FORCE_STORAGE_REQUIREMENTS : ANALYZE_STORAGE_REQUIREMENTS, @@ -270,6 +286,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { 'analyze launch: forwarding route configuration to the worker', ); + const analysisOptions = { + ...opts, + graphPublication: 'auto' as const, + resources, + routeHookConfig, + }; + jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); // ── Worker fork with auto-retry ────────────────────────────── @@ -281,20 +304,12 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] : []; - // Worker heap: 8192MB historical default, but never above what this - // machine/container actually has (#2649 review — a fixed 8192 inside a - // smaller cgroup limit died to the kernel with a misleading remedy). - // GITNEXUS_SERVER_ANALYZE_HEAP_MB overrides as an absolute value. - const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); - const workerHeapMb = - Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb : Math.min(8192, autoHeapCapMb()); - const launchAborted = (jobId: string): boolean => { const current = jobManager.getJob(jobId); return !current || isTerminalJobStatus(current.status) || jobManager.hasPendingCancel(jobId); }; - const forkWorker = () => { + const forkWorker = (continuation?: ContinueMessage) => { if (launchAborted(job.id)) { // Cancelled (or timed out) between lock acquisition and the fork, or // during a crash-retry delay. A pending-cancel job stays non-terminal @@ -305,7 +320,11 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { } const child = fork(workerPath, [], { - execArgv: [...tsxHookArgs, `--max-old-space-size=${workerHeapMb}`], + execArgv: [ + ...tsxHookArgs, + `--max-old-space-size=${resources.mainHeapMb - 48}`, + '--max-semi-space-size=16', + ], stdio: ['ignore', 'pipe', 'pipe', 'ipc'], }); @@ -325,12 +344,15 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // "don't release on cancel error" would leak the lock. let holdLockUntilExit = false; let childExited = false; + let graphVisible = false; + let prepared: GraphPreparedMessage | undefined; + let terminalMessage: Extract | undefined; child.stderr?.on('data', (chunk: Buffer) => { stderrChunks += chunk.toString(); if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); }); - child.on('message', (msg: WorkerMessage) => { + const handleMessage = (msg: WorkerMessage) => { // Ignore any message once the job is terminal — a late worker message (a // SIGTERM-driven `error` after `complete`, or vice versa) must not // re-release the repo lock or flip the reported status. Mirrors the `exit` @@ -338,12 +360,64 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { const current = jobManager.getJob(job.id); if (!current || isTerminalJobStatus(current.status)) return; + if ((msg.type === 'complete' || msg.type === 'error') && !childExited) { + terminalIpcSeen = true; + terminalMessage ??= msg; + if (jobManager.applyPendingCancel(job.id)) holdLockUntilExit = true; + return; + } + if (msg.type === 'graph-prepared') { + try { + if (prepared || continuation) throw new Error('Repeated graph preparation'); + prepared = { ...msg, handoff: projectPreparedAnalysisHandoff(msg.handoff) }; + } catch (error) { + terminalMessage = { type: 'error', message: (error as Error).message }; + terminalIpcSeen = true; + child.send({ type: 'cancel' }); + } + return; + } + if (msg.type === 'graph-ready') { + // Core has closed the writer and verified publication. Refresh readers + // while this job is still active, then admit the independent vector phase. + void closeDbHandle() + .then(() => backend.init()) + .then(() => { + if (childExited) return; + if (launchAborted(job.id)) child.send({ type: 'cancel' }); + else { + graphVisible = true; + child.send({ type: 'graph-visible' }); + } + }) + .catch((error) => { + if (childExited) return; + terminalMessage = { + type: 'error', + message: `Server failed to reload the published graph: ${(error as Error).message}`, + }; + terminalIpcSeen = true; + try { + child.send({ type: 'cancel' }); + } catch { + /* Wait for actual exit. */ + } + }); + return; + } + if (msg.type === 'complete' || msg.type === 'error') terminalIpcSeen = true; if (msg.type === 'progress') { jobManager.updateJob(job.id, { status: 'analyzing', - progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, + progress: { + ...projectAnalysisPhaseProgress(current.progress), + ...projectAnalysisPhaseProgress(msg), + phase: msg.phase, + percent: msg.percent, + message: msg.message, + }, }); } else if (msg.type === 'complete') { if (jobManager.applyPendingCancel(job.id)) { @@ -374,20 +448,21 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // would spin the full 60s and hold the single global analyze slot. // ftsRepairedOnly DOES rewrite `lbug` (initLbug + createSearchFTSIndexes) // so it still waits. - const settle = msg.result.alreadyUpToDate - ? Promise.resolve(true) - : waitForSettledIndex( - // A checkout's first analyze may write a shared-store slot the - // launcher's pre-run lookup could not see yet (#3352); the - // worker reports it. - (msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath) - ? msg.result.storagePath - : null) ?? analyzeLockKey, - jobStartMs, - opts.branch, - msg.result.isPrimaryBranch, - () => launchAborted(job.id), - ); + const settle = + continuation || graphVisible || msg.result.alreadyUpToDate + ? Promise.resolve(true) + : waitForSettledIndex( + // A checkout's first analyze may write a shared-store slot the + // launcher's pre-run lookup could not see yet (#3352); the + // worker reports it. + (msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath) + ? msg.result.storagePath + : null) ?? analyzeLockKey, + jobStartMs, + opts.branch, + msg.result.isPrimaryBranch, + () => launchAborted(job.id), + ); settle .then((settled) => { if (launchAborted(job.id)) { @@ -508,7 +583,8 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { jobManager.updateJob(job.id, { status: 'failed', error: msg.message }); } } - }); + }; + child.on('message', handleMessage); child.on('error', (err) => { // Fake test children have no `pid`. Treat that as pre-spawn so the @@ -532,6 +608,18 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { child.on('exit', (code) => { childExited = true; + if (terminalMessage) { + handleMessage( + code === 0 + ? terminalMessage + : { + type: 'error', + message: `Analyze worker exited unsuccessfully (${code ?? 'signal'})`, + }, + ); + if (launchAborted(job.id)) releaseLockOnce(); + return; + } const j = jobManager.getJob(job.id); if (!j || isTerminalJobStatus(j.status) || jobManager.hasPendingCancel(job.id)) { // (a) complete/error IPC is in flight (`terminalIpcSeen`) — that @@ -553,6 +641,17 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { // finished, successful analysis. if (terminalIpcSeen) return; + if (prepared && code === 0 && child.pid) { + forkWorker({ + type: 'continue', + repoPath: targetPath, + options: analysisOptions, + writerPid: child.pid, + prepared, + }); + return; + } + // Worker crashed — attempt retry if under the limit if (j.retryCount < MAX_WORKER_RETRIES) { j.retryCount++; @@ -571,7 +670,7 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { }, }); stderrChunks = ''; - setTimeout(forkWorker, delay); + setTimeout(() => forkWorker(), delay); } else { // Exhausted retries — permanent failure releaseLockOnce(); @@ -586,20 +685,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { jobManager.registerChild(job.id, child); // Send start command to child - child.send({ - type: 'start', - repoPath: targetPath, - options: { - force: !!opts.force, - embeddings: !!opts.embeddings, - dropEmbeddings: !!opts.dropEmbeddings, - ...(opts.springActuatorPath ? { springActuatorPath: opts.springActuatorPath } : {}), - ...(opts.asyncApiSpecPath ? { asyncApiSpecPath: opts.asyncApiSpecPath } : {}), - ...(opts.registryName ? { registryName: opts.registryName } : {}), - ...(opts.branch ? { branch: opts.branch } : {}), - routeHookConfig, + child.send( + continuation ?? { + type: 'start', + repoPath: targetPath, + options: analysisOptions, }, - }); + ); }; try { diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index ddda81a83..825b0a5b3 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -13,19 +13,24 @@ * worker — only the lightweight `analyze-worker-ipc` projection helper. */ import type { AnalyzeOptions } from '../core/run-analyze.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; // The IPC message protocol lives in a declarations-only leaf, NOT in the // `analyze-worker.ts` entry module: importing it from the entry made this // module depend on the very module that depends on it (#cycle). Erased at // runtime either way; this way the graph is acyclic too. -import type { WorkerMessage } from './analyze-worker-protocol.js'; +import type { WorkerMessage, ContinueMessage } from './analyze-worker-protocol.js'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; +import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; // Value import (instanceof): index-lock is a lightweight storage primitive // (node:fs/net/crypto only), so this does NOT pull in run-analyze/repo-manager. import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index-lock.js'; export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; + continuePreparedAnalysis?: typeof import('../core/run-analyze.js').continuePreparedAnalysis; + onGraphPublished?: () => Promise; + signal?: AbortSignal; assertAnalysisFinalized: typeof import('../storage/repo-manager.js').assertAnalysisFinalized; send: (msg: WorkerMessage) => void; /** @@ -35,6 +40,9 @@ export interface WorkerAnalysisDeps { * both report a terminal outcome (#2264 P3). See {@link createTerminalClaim}. */ claimTerminal: () => boolean; + /** Required for nonterminal graph handoff; never claim the final outcome just + * to test whether cancellation already owns it. */ + isCancelled?: () => boolean; } /** @@ -48,25 +56,66 @@ export async function runWorkerAnalysis( options: AnalyzeOptions, deps: WorkerAnalysisDeps, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, + continuation?: ContinueMessage, ): Promise { let terminal: WorkerMessage; try { const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap ? [runnerIdentityAtBootstrap] : []; - const result = await deps.runFullAnalysis( - repoPath, - // This worker force-exits right after reporting, so skip the native close - // (it can double-free in LadybugDB's ClientContext destructor after --pdg - // writes); flushWAL still persists the index, process.exit reclaims handles. - { ...options, skipNativeCloseOnExit: true }, - { - onProgress: (phase, percent, message) => - deps.send({ type: 'progress', phase, percent, message }), - onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), - }, - ...bootstrapArgs, - ); + const callbacks: import('../core/run-analyze.js').AnalyzeCallbacks = { + signal: deps.signal, + onGraphPublished: deps.onGraphPublished, + onProgress: (phase, percent, message, details) => + deps.send({ + type: 'progress', + phase, + percent, + message, + ...projectAnalysisPhaseProgress(details), + }), + onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), + }; + let result: import('../core/run-analyze.js').AnalyzeResult; + if (continuation) { + if (!deps.continuePreparedAnalysis) throw new Error('Missing analysis continuation'); + const handoff = projectPreparedAnalysisHandoff(continuation.prepared.handoff); + if (handoff.repoPath !== repoPath || continuation.prepared.result.repoPath !== repoPath) + throw new Error('Continuation belongs to a different repository'); + const metadata = await deps.continuePreparedAnalysis( + handoff, + continuation.writerPid, + options, + callbacks, + deps.signal, + ); + result = { + ...continuation.prepared.result, + stats: metadata.stats ?? {}, + rebuildReasons: [], + }; + } else { + // Native allocations belong to this exit-only graph process. + result = await deps.runFullAnalysis( + repoPath, + { ...options, skipNativeCloseOnExit: true }, + callbacks, + ...bootstrapArgs, + ); + } + if (result.preparedGraph) { + if (!deps.isCancelled) throw new Error('Graph handoff requires cancellation state'); + if (result.graphWriteCollapsed) + throw new Error('Collapsed graph cannot be prepared for publication'); + if (!deps.isCancelled()) { + deps.send({ + type: 'graph-prepared', + handoff: projectPreparedAnalysisHandoff(result.preparedGraph), + result: projectAnalyzeResultForIpc(result), + }); + } + return; + } // P2 (#2264): a half-finalized repo — meta.json written but the global // registry entry missing (e.g. a prior collision-aborted run, or a wiped // registry) — must NOT be reported as a successful analysis. Mirror the CLI's diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 7b046bbe3..3e6991ffb 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -24,7 +24,9 @@ * sites (`analyze-launch.ts`, tests) keep working unchanged. */ import type { AnalyzeOptions } from '../core/run-analyze.js'; +import type { AnalysisPhaseProgress } from '../core/analysis-progress.js'; import type { AnalyzeResultIpc } from './analyze-worker-ipc.js'; +import type { PreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js'; /** Parent → child: start one analysis run. */ export interface StartMessage { @@ -33,14 +35,27 @@ export interface StartMessage { options: AnalyzeOptions; } +/** A new process may continue only after the graph writer actually exited. */ +export interface ContinueMessage { + type: 'continue'; + repoPath: string; + options: AnalyzeOptions; + writerPid: number; + prepared: GraphPreparedMessage; +} + +export interface GraphVisibleMessage { + type: 'graph-visible'; +} + /** Parent → child: request safe cancellation at the next JS-visible checkpoint. */ export interface CancelMessage { type: 'cancel'; } -export type ParentMessage = StartMessage | CancelMessage; +export type ParentMessage = StartMessage | ContinueMessage | CancelMessage | GraphVisibleMessage; -export interface ProgressMessage { +export interface ProgressMessage extends AnalysisPhaseProgress { type: 'progress'; phase: string; percent: number; @@ -54,6 +69,18 @@ export interface CompleteMessage { result: AnalyzeResultIpc; } +/** Nonterminal: the graph process still owns its native allocations. */ +export interface GraphPreparedMessage { + type: 'graph-prepared'; + handoff: PreparedAnalysisHandoff; + result: AnalyzeResultIpc; +} + +/** Wait for the serving parent to refresh readers before making paid requests. */ +export interface GraphReadyMessage { + type: 'graph-ready'; +} + export interface ErrorMessage { type: 'error'; message: string; @@ -75,4 +102,9 @@ export interface ErrorMessage { } /** Child → parent IPC messages. Shared with the parent-side launcher. */ -export type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage; +export type WorkerMessage = + | ProgressMessage + | GraphPreparedMessage + | GraphReadyMessage + | CompleteMessage + | ErrorMessage; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index 766cf0022..ff33526ae 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -74,10 +74,38 @@ process.on('unhandledRejection', (reason: unknown) => { // returned to JS. SIGTERM is retained only for local process shutdown. let cancellationRequested = false; let started = false; +const cancellation = new AbortController(); +let graphVisible: (() => void) | undefined; + +async function waitForGraphVisibility(): Promise { + cancellation.signal.throwIfAborted(); + await new Promise((resolve, reject) => { + const cleanup = () => { + graphVisible = undefined; + cancellation.signal.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); + reject(cancellation.signal.reason); + }; + graphVisible = () => { + cleanup(); + resolve(); + }; + cancellation.signal.addEventListener('abort', onAbort, { once: true }); + try { + send({ type: 'graph-ready' }); + } catch (error) { + cleanup(); + reject(error); + } + }); +} function requestWorkerCancellation(source: string): void { if (cancellationRequested) return; cancellationRequested = true; - if (claimTerminal()) { + cancellation.abort(new Error(`Analysis cancelled (${source})`)); + if (claimTerminal() && process.connected) { send({ type: 'error', message: `Analysis cancelled (${source})` }); } if (!started) { @@ -102,9 +130,14 @@ function exitAfterCancellation(): void { } process.on('SIGTERM', () => requestWorkerCancellation('worker received SIGTERM')); +process.on('disconnect', () => requestWorkerCancellation('supervisor disconnected')); // Listen for parent commands — guarded against re-entry. process.on('message', async (msg: ParentMessage) => { + if (msg.type === 'graph-visible') { + graphVisible?.(); + return; + } if (msg.type === 'cancel') { requestWorkerCancellation('parent requested cancellation'); return; @@ -141,11 +174,16 @@ process.on('message', async (msg: ParentMessage) => { msg.options, { runFullAnalysis: prepared.loaded.analysisModule.runFullAnalysis, + continuePreparedAnalysis: prepared.loaded.analysisModule.continuePreparedAnalysis, + onGraphPublished: msg.options.graphPublication ? waitForGraphVisibility : undefined, + signal: cancellation.signal, assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized, send, claimTerminal, + isCancelled: () => cancellationRequested, }, prepared.runnerIdentity, + msg.type === 'continue' ? msg : undefined, ); } catch (error) { if (claimTerminal()) { diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 5a819edcf..eb91acbf8 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -9,14 +9,16 @@ */ import { - acquireIndexLock, IndexLockTimeoutError, requireExclusiveIndexLock, sweepStagingArtifacts, type IndexLockHandle, } from '../storage/index-lock.js'; +import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; import { resolveGraphPath } from '../storage/shared-store.js'; +import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js'; +import { readIndexCapabilities } from '../storage/index-capabilities.js'; import { reclaimAfterSlotRemoval, removeCheckoutStorage, @@ -29,7 +31,6 @@ import { canonicalizePath, cloneDirBelongsToEntry, loadMeta, - saveMeta, listRegisteredRepos, registryPathEquals, type RegistryEntry, @@ -44,10 +45,9 @@ import { import { executeQuery, executePrepared, - executeWithReusedStatement, streamQuery, - flushWAL, closeLbug, + drainAnalysisNativeResources, withLbugDb, isReadOnlyDbError, } from '../core/lbug/lbug-adapter.js'; @@ -62,24 +62,13 @@ import { contentRetentionFromMeta, isFullSourceAvailable, } from '../core/content-retention.js'; -import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { getFtsDisabledReason, type FtsDisabledReason } from '../core/search/fts-policy.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js'; import { fileURLToPath } from 'url'; -import { isTerminalJobStatus, JobManager, type AnalyzeJobPartialOutcome } from './analyze-job.js'; +import { isTerminalJobStatus, JobManager } from './analyze-job.js'; import { mountSSEProgress } from './sse-progress.js'; -import { - resolveEmbedRunOutcome, - withMeasuredEmbeddingCount, - type EmbedRunFinalizeContext, -} from './embed-run-outcome.js'; -import { decideEmbeddingResume, mintInterruptedCheckpoint } from '../core/embedding-checkpoint.js'; -import { - measurePersistedEmbeddingCount, - persistedEmbeddingCountOrUndefined, - type PersistedEmbeddingCount, -} from '../core/embedding-count.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; import { assertString, BadRequestError, createRouteLimiter } from './validation.js'; import { parseGrepQuery, GREP_TIME_BUDGET_MS } from './grep-params.js'; import { runGrepScanInWorker } from './grep-scan.js'; @@ -997,6 +986,8 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const app = express(); app.disable('x-powered-by'); const serverStartedAt = Date.now(); + let shuttingDown = false; + const activeEmbeddingRuns = new Set>(); // Which upstream hops may set X-Forwarded-*. Process-wide: every route's // req.ip, and so the per-IP rate limiter, resolves through this. @@ -1065,6 +1056,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const activeRepoPaths = new Set(); const acquireRepoLock = (repoPath: string): string | null => { + if (shuttingDown) return 'Server is shutting down'; if (activeRepoPaths.has(repoPath)) { return `Another job is already active for this repository`; } @@ -1152,20 +1144,28 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) { if (clientGone) return null; // client disconnected — stop polling const currentJob = jobManager.getJob(job.id); - if (!currentJob || currentJob.status === 'failed') { + if ( + currentJob?.progress.graphPhase === 'ready' || + currentJob?.status === 'complete' || + currentJob?.status === 'failed' + ) { + // A published graph is usable while embeddings are pending or + // failed. Re-resolve its owned registry entry before returning. + await backend.init(); + const freshRepos = await listRegisteredRepos({ + validate: options.validateStorage !== false, + }); + const published = resolveRegisteredRepoEntry(freshRepos, repoName); + if (published) return validate(published); + if (currentJob.status === 'complete' || currentJob.status === 'failed') return null; + } + if (!currentJob) { // The job is over and produced no registry entry. This is a // plain "not found", not "still analyzing" — falling through to // the timed-out sentinel here told callers to keep waiting for // a job that had already failed. return null; } - if (currentJob.status === 'complete') { - await backend.init(); - const freshRepos = await listRegisteredRepos({ - validate: options.validateStorage !== false, - }); - return validate(resolveRegisteredRepoEntry(freshRepos, repoName)); - } await new Promise((r) => setTimeout(r, 1000)); } // Timed out — signal to the caller with a specific message @@ -1270,15 +1270,24 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // Timed out waiting for an active analysis job if (respondIfAnalysisPending(entry, res)) return; const meta = await loadMeta(entry.storagePath); - const [staleness, availability] = await Promise.all([ + const [staleness, availability, embeddingProgress, capabilities] = await Promise.all([ checkStalenessAsync(entry.path, resolveLastCommit(entry, meta)), getSourceAvailability(entry, meta), + readEmbeddingJobProgress(entry.storagePath, meta), + readIndexCapabilities(entry.storagePath, meta), ]); res.json( - projectRepoDetail(entry, meta, staleness, { - contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta), - sourceAvailable: availability.available, - }), + projectRepoDetail( + entry, + meta, + staleness, + { + contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta), + sourceAvailable: availability.available, + }, + embeddingProgress, + capabilities, + ), ); } catch (err: any) { if (sendStorageRequirementHttp(err, res)) return; @@ -1510,15 +1519,33 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => async () => { let searchResults: any[]; let ftsAvailable: boolean | undefined; + const metadata = mode === 'bm25' ? null : await loadMeta(entry.storagePath); + const snapshot = + metadata && + (metadata.graphGeneration || metadata.vectorGeneration || metadata.embeddingJob) + ? { metadataDirectory: entry.storagePath, metadata, graphPath: lbugPath } + : undefined; + const semanticReady = + mode !== 'bm25' && (await import('../core/embeddings/embedder.js')).isEmbedderReady(); + const semanticLane = async (execute: typeof executeQuery, text: string, count = 10) => { + const { semanticSearch } = await import('../core/embeddings/embedding-pipeline.js'); + return semanticSearch( + execute, + text, + count, + undefined, + snapshot, + snapshot + ? async (input) => (await import('../mcp/core/embedder.js')).embedQuery(input) + : undefined, + ); + }; if (mode === 'semantic') { - const { isEmbedderReady } = await import('../core/embeddings/embedder.js'); - if (!isEmbedderReady()) { + if (!semanticReady) { return { searchResults: [] as any[], ftsAvailable: undefined }; } - const { semanticSearch: semSearch } = - await import('../core/embeddings/embedding-pipeline.js'); - searchResults = await semSearch(executeQuery, query, limit); + searchResults = await semanticLane(executeQuery, query, limit); // Normalize semantic results to HybridSearchResult shape searchResults = searchResults.map((r: any, i: number) => ({ ...r, @@ -1536,15 +1563,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => })); } else { // hybrid (default) - const { isEmbedderReady } = await import('../core/embeddings/embedder.js'); - if (isEmbedderReady()) { - const { semanticSearch: semSearch } = - await import('../core/embeddings/embedding-pipeline.js'); + if (semanticReady) { searchResults = await hybridSearch( query, limit, executeQuery, - semSearch, + semanticLane, ftsDisabledReason, ); if (ftsDisabledReason) ftsAvailable = false; @@ -2033,7 +2057,6 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }); } })(); - res.status(202).json({ jobId: job.id, status: job.status }); } catch (err: any) { if (err.message?.includes('already in progress')) { @@ -2144,266 +2167,86 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }, EMBED_TIMEOUT_MS); // Run embedding pipeline asynchronously - (async () => { - // Set inside withLbugDb, read after it closes (#2790). - let partialRunError: string | undefined; - let partialRunDetail: AnalyzeJobPartialOutcome | undefined; - // The in-memory repo lock only serializes this server; a CLI analyze - // in another process guards the slot with the index lock, so hold it - // for the whole embedding write, released in the finally below. + const run = (async () => { + // Both server writers and other CLI processes serialize on this slot. let slotLock: IndexLockHandle | undefined; try { - slotLock = await acquireIndexLock(storagePath, { sweep: false }); + slotLock = await acquireAnalysisSlotLock(storagePath, { sweep: false }); requireExclusiveIndexLock( slotLock, `Cannot acquire the index lock at ${storagePath}; refusing an unlocked embedding run.`, ); - // This writer cannot recover staged generations. Preserve their - // receipts, including malformed ones, before sweeping or writing. - const recoveryCheckpoint = (await loadMeta(storagePath))?.embeddingCheckpoint; - if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) { + embedController.signal.throwIfAborted(); + const metadata = await loadMeta(storagePath); + if ( + metadata?.embeddingCheckpoint && + Object.hasOwn(metadata.embeddingCheckpoint, 'recovery') + ) throw new Error( - 'Cannot generate embeddings: the index checkpoint references staged embeddings. ' + - 'Run `gitnexus analyze` to recover them first.', + 'Cannot generate embeddings: the index checkpoint references staged embeddings. Run `gitnexus analyze` to recover them first.', ); - } + const { readPreparedAnalysis } = await import('../core/analysis-publication.js'); + if (await readPreparedAnalysis(storagePath)) + throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.'); sweepStagingArtifacts(storagePath); - // Writes go to the slot's own graph; a shared-store checkout - // reading an immutable commit graph (#3352) takes a private copy. - if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) { + const { resolveHostAnalysisResources } = + await import('../core/analysis-host-resources.js'); + const resources = resolveHostAnalysisResources({ + includeParsing: false, + includeEmbeddings: true, + }); + if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) throw new Error('The shared graph this repository reads is gone. Re-run analyze.'); - } - const lbugPath = path.join(storagePath, LBUG_DIRECTORY); - const ftsSession = await loadFtsSession(storagePath); - let embeddingMeta = ftsSession.meta; - // Writable embed still replays a leftover FTS-abort WAL. Refuse - // here — doInitLbug only gates the readOnly path, and analyze - // writers must still be able to park/rebuild. + const lbugPath = resolveGraphPath(storagePath); await assertReadOnlyFtsCrashSafe(lbugPath); - await withLbugDb( - lbugPath, - async () => { - const { runEmbeddingPipeline } = - await import('../core/embeddings/embedding-pipeline.js'); - const { resolveEmbeddingIdentity } = - await import('../core/embeddings/embedding-identity.js'); - const embeddingIdentity = resolveEmbeddingIdentity(); - if (!embeddingMeta) { - throw new Error('Repository metadata is missing; run gitnexus analyze first'); - } - const priorCheckpoint = embeddingMeta.embeddingCheckpoint; - // The SAME decision the CLI's resume gate makes - // (core/embedding-checkpoint.ts). This route used to hard-throw on - // any identity mismatch and ignore `attempts` entirely, so a - // `'partial'` marker written by `gitnexus analyze` and resumed - // here hit exactly the permanent wedge `kind` exists to remove: - // two readers of one record disagreeing about the rule it encodes. - // No `force`/`--drop-embeddings` equivalent exists on this route, - // so the flag options go unset and `'discard'` is unreachable — - // it is folded into the abandon arm rather than given an invented - // flag. `maxAttempts` is left to the shared default. - const resume = priorCheckpoint - ? decideEmbeddingResume(priorCheckpoint, embeddingIdentity) - : undefined; - if (resume?.action === 'abort') throw new Error(resume.error); - if (resume?.action === 'abandon' || resume?.action === 'discard') { - logger.warn({ repo: entry.name }, resume.log); - } - const forceReembedNodeIds: ReadonlySet = - resume?.action === 'resume' ? resume.pendingNodeIds : new Set(); - const saveEmbeddingCheckpoint = async ( - checkpoint: { - nodesProcessed: number; - totalNodes: number; - chunksProcessed: number; - }, - pendingNodeIds: string[], - embeddings?: PersistedEmbeddingCount, - ): Promise => { - // tri-review NEW-2: re-read immediately before writing (mirrors - // the pattern in run-analyze.ts's --repair-fts stamp) instead of - // spreading the stale `embeddingMeta` snapshot captured once at - // job start. This job can run up to EMBED_TIMEOUT_MS (30 min); - // without a fresh read, a concurrent writer's update (e.g. a - // --repair-fts capability stamp) would be silently reverted on - // every checkpoint save for the job's whole lifetime. - const latestMeta = (await loadMeta(storagePath)) ?? embeddingMeta; - // `stats.embeddings` only moves when the caller MEASURED the - // live count (the post-flush `onCheckpoint`). The window-start - // callback measures nothing and passes nothing: restating the - // old count there would re-publish a stale number and clobber - // what a preceding `onCheckpoint` just wrote (same split as - // run-analyze.ts's checkpoint writer). - embeddingMeta = withMeasuredEmbeddingCount( - { - ...latestMeta, - // In flight ⇒ `kind: 'interrupted'` (embedding-checkpoint.ts). - embeddingCheckpoint: mintInterruptedCheckpoint( - embeddingIdentity, - checkpoint, - pendingNodeIds, - ), + await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); + const { runAnalysisEmbeddingPhase } = + await import('../core/analysis-embedding-phase.js'); + await runAnalysisEmbeddingPhase({ + metadataDirectory: storagePath, + graphPath: lbugPath, + resources, + nodeLimit: 0, + expectedGeneration: metadata?.graphGeneration?.id, + signal: embedController.signal, + callbacks: { + onGraphPublished: async () => { + await drainAnalysisNativeResources(); + await backend.init(); + }, + onProgress: (phase, percent, message, details) => { + embedJobManager.updateJob(job.id, { + progress: { + ...projectAnalysisPhaseProgress(details), + phase: + phase === 'embeddings' + ? details?.embeddingProgress?.phase === 'indexing' + ? 'indexing' + : 'embedding' + : phase, + percent, + message, }, - embeddings, - ); - await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); - await saveMeta(storagePath, embeddingMeta); - }; - /** - * Count the persisted rows, or report the answer never arrived. - * The TRI-STATE is carried to the fold rather than collapsed here: - * `unknown` is not 0, and only the fold knows what to carry - * forward instead (core/embedding-count.ts). - */ - const countPersistedEmbeddings = async (): Promise => { - const counted = await measurePersistedEmbeddingCount(executeQuery); - if (counted.kind === 'unknown') { - logger.warn( - { reason: counted.reason }, - '[embed] could not count persisted embeddings; leaving stats.embeddings untouched', - ); - } - return counted; - }; - // Fetch existing content hashes for incremental embedding. - // Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling. - const { fetchExistingEmbeddingHashes } = - await import('../core/lbug/lbug-adapter.js'); - const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery); - if (existingEmbeddings && existingEmbeddings.size > 0) { - console.log( - `[embed] ${existingEmbeddings.size} nodes already embedded — incremental run with content-hash comparison`, - ); - } - const pipelineResult = await runEmbeddingPipeline( - executeQuery, - executeWithReusedStatement, - (p) => { - embedJobManager.updateJob(job.id, { - progress: { - // `ready` maps to 'finalizing', NOT 'complete' (#2790). - // The pipeline emits `ready`/100% unconditionally before - // returning — including when it dropped nodes to endpoint - // failures — and the route has not measured the index or - // decided the outcome yet, so 'complete' here would make - // the job record contradict itself (`status: 'analyzing'`, - // `progress.phase: 'complete'`). - phase: - p.phase === 'ready' - ? 'finalizing' - : p.phase === 'error' - ? 'failed' - : p.phase, - percent: p.percent, - message: - p.phase === 'loading-model' - ? 'Loading embedding model...' - : p.phase === 'embedding' - ? `Embedding nodes (${p.percent}%)...` - : p.phase === 'indexing' - ? 'Creating vector index...' - : p.phase === 'ready' - ? 'Finalizing embeddings...' - : `${p.phase} (${p.percent}%)`, - }, - }); - }, - {}, // config: use defaults - undefined, // skipNodeIds - existingEmbeddings, - { - source: { - repoPath: entry.path, - fileHashes: embeddingMeta.fileHashes, - contentRetention: contentRetentionFromMeta(embeddingMeta), - }, - signal: embedController.signal, - forceReembedNodeIds, - onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { - await saveEmbeddingCheckpoint(checkpoint, nodeIds); - }, - onCheckpoint: async (checkpoint) => { - // Count AFTER the flush, so the number describes rows that - // are durable rather than rows still pending in the WAL. - await flushWAL(); - await saveEmbeddingCheckpoint( - checkpoint, - [], - await countPersistedEmbeddings(), - ); - }, - }, - ); - - // Flush WAL so subsequent /api/search requests see the new - // embeddings immediately (#1149). In the CLI path closeLbug() - // handles this during process exit, but the server keeps the - // connection open for other routes — a CHECKPOINT is enough. - await flushWAL(); - // Measure inside withLbugDb, after the flush and while the - // connection is still open — this is the route's only chance to - // stamp `stats.embeddings` (embed-run-outcome.ts). A partial run - // gets the same stamp: an honest count of a partial index is what - // makes it survivable. - const measuredEmbeddings = await countPersistedEmbeddings(); - // Same re-read-before-write reasoning as saveEmbeddingCheckpoint - // above — and the outcome decision reads it too: its - // `embeddingCheckpoint` is the marker this run's own mid-run - // writer saved, which is the only record of the work when the - // count query could not answer. - const finalMeta = (await loadMeta(storagePath)) ?? embeddingMeta; - const finalizeContext: EmbedRunFinalizeContext = { - measuredEmbeddings: persistedEmbeddingCountOrUndefined(measuredEmbeddings), - onDisk: finalMeta, - // The marker the job STARTED from — `finalMeta`'s has since been - // overwritten by the in-flight writer, so only this one carries - // the `'partial'` attempt chain. - resumedFrom: priorCheckpoint, - }; - const outcome = resolveEmbedRunOutcome( - embeddingIdentity, - pipelineResult, - finalizeContext, - ); - partialRunError = outcome.error; - partialRunDetail = outcome.partial; - embeddingMeta = withMeasuredEmbeddingCount( - { ...finalMeta, embeddingCheckpoint: outcome.checkpoint }, - measuredEmbeddings, - ); - await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS); - await saveMeta(storagePath, embeddingMeta); + }); + }, + onLog: (message) => logger.info({ repo: entry.name }, message), }, - skipFtsOption(ftsSession.skipFts), - ); - - // Don't overwrite 'failed' if the job was cancelled while the pipeline was running + }); + await backend.init(); const current = embedJobManager.getJob(job.id); - if (!current || current.status !== 'failed') { - // The ONLY terminal event for the job, on both branches — nothing - // earlier maps to a terminal status, and `updateJob` synthesizes - // exactly one event per terminal status (#2264 P3). The explicit - // `progress` keeps the record self-consistent for a poller reading - // `progress.phase` (#2790). - embedJobManager.updateJob( - job.id, - partialRunError === undefined - ? { - status: 'complete', - progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' }, - } - : { - status: 'failed', - error: partialRunError, - // Lets a client separate "retry these N nodes" from "the - // run produced nothing" without a new status member. - partial: partialRunDetail, - progress: { phase: 'failed', percent: 100, message: partialRunError }, - }, - ); - } + if (current && !isTerminalJobStatus(current.status)) + embedJobManager.updateJob(job.id, { + status: 'complete', + progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' }, + }); } catch (err: any) { const current = embedJobManager.getJob(job.id); + // Provider causes may contain response text. Log only the public + // message, as the CLI does, without pino's recursive Error serializer. + logger.error( + { reason: err.message || 'Embedding generation failed', repo: entry.name }, + 'Embedding generation failed', + ); if (!current || current.status !== 'failed') { embedJobManager.updateJob(job.id, { status: 'failed', @@ -2416,6 +2259,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => releaseRepoLock(repoLockPath); } })(); + activeEmbeddingRuns.add(run); + // Observe both outcomes without creating an unhandled finally promise. + void run.then( + () => activeEmbeddingRuns.delete(run), + () => activeEmbeddingRuns.delete(run), + ); res.status(202).json({ jobId: job.id, status: 'analyzing' }); } catch (err: any) { @@ -2554,11 +2403,17 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // destination is `sync: false` (buffered); `flushLoggerSync()` before // `process.exit` so records emitted during cleanup reach stderr. const shutdown = async () => { + if (shuttingDown) return; + shuttingDown = true; console.log('\nShutting down...'); updateController.stop(); server.close(); jobManager.dispose(); embedJobManager.dispose(); + // Abort requests are cooperative. Keep the ordinary writer locks and + // native handles alive until each in-process embedding run has settled. + await Promise.allSettled(activeEmbeddingRuns); + await jobManager.waitForChildren(); await cleanupMcp(); await closeLbug(); await backend.disconnect(); @@ -2576,12 +2431,10 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // Worker-thread transports (pino-pretty under TTY) handle their own // flush on process exit in v10. `pino.final` was removed in v10 // because the new transport architecture made it unnecessary. - let shuttingDown = false; process.on('uncaughtException', (err) => { logger.error({ err }, 'GitNexus uncaughtException'); flushLoggerSync(); if (!shuttingDown) { - shuttingDown = true; shutdown().catch(() => {}); } }); diff --git a/gitnexus/src/server/ops-snapshot.ts b/gitnexus/src/server/ops-snapshot.ts index 18ed36f2d..4ad9c0ede 100644 --- a/gitnexus/src/server/ops-snapshot.ts +++ b/gitnexus/src/server/ops-snapshot.ts @@ -13,6 +13,7 @@ import { } from './analyze-job.js'; import { publicRepoId } from './public-repo-id.js'; import { escapeRegExp } from './validation.js'; +import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js'; export interface OpsJobView { id: string; @@ -227,11 +228,14 @@ export const redactPublicText = (text: string, known?: Array export const publicOpsProgress = ( progress: AnalyzeJobProgress, known?: Array, -): AnalyzeJobProgress => ({ - phase: progress.phase, - percent: progress.percent, - message: redactPublicText(progress.message, known), -}); +): AnalyzeJobProgress => { + return { + phase: progress.phase, + percent: progress.percent, + message: redactPublicText(progress.message, known), + ...projectAnalysisPhaseProgress(progress), + }; +}; export const serializeOpsJob = ( job: AnalyzeJob, diff --git a/gitnexus/src/server/repo-projection.ts b/gitnexus/src/server/repo-projection.ts index bfb0171f4..7605a3d86 100644 --- a/gitnexus/src/server/repo-projection.ts +++ b/gitnexus/src/server/repo-projection.ts @@ -17,6 +17,8 @@ import { import type { ContentRetention, RepoMeta } from '../storage/repo-meta.js'; import type { RegistryEntry } from '../storage/repo-manager.js'; import { publicRepoId } from './public-repo-id.js'; +import { getGraphIncompleteReasons, getIndexIncompleteReasons } from '../core/index-freshness.js'; +import type { EmbeddingJobProgress } from '../core/analysis-progress.js'; /** Retention + checkout facts computed by the route (see getSourceAvailability). */ export interface RepoProjectionSource { @@ -85,6 +87,8 @@ export const projectRepoDetail = ( meta: RepoMeta | null | undefined, staleness: StalenessInfo, source: RepoProjectionSource, + embeddingProgress?: EmbeddingJobProgress, + capabilities?: RepoMeta['capabilities'], ) => ({ name: entry.name, repoPath: entry.path, @@ -95,6 +99,10 @@ export const projectRepoDetail = ( branch: meta?.branch ?? entry.branch, contentRetention: source.contentRetention, sourceAvailable: source.sourceAvailable, + incompleteReasons: getIndexIncompleteReasons(meta), + graphIncompleteReasons: getGraphIncompleteReasons(meta), + embeddingProgress, + capabilities, ...stalenessField(staleness), }); diff --git a/gitnexus/src/storage/analysis-artifact-cleanup.ts b/gitnexus/src/storage/analysis-artifact-cleanup.ts new file mode 100644 index 000000000..7d121dbfc --- /dev/null +++ b/gitnexus/src/storage/analysis-artifact-cleanup.ts @@ -0,0 +1,134 @@ +/** Reclaim only unreferenced owned artifacts under the ordinary slot lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { RepoMeta } from './repo-meta.js'; +import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js'; +import { readEmbeddingArtifactOwners } from './embedding-artifact.js'; +import { readLegacyEmbeddingSource } from './legacy-embedding-source.js'; +import { logger } from '../core/logger.js'; + +const GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/; +const ATTEMPT_METADATA = /^analysis-meta-[a-f0-9]{32}\.json$/; +const ID = /^[a-f0-9]{64}$/; + +/** Keep the published and previous graph, all model spaces, pending jobs, and + * their paid reuse sources. Windows busy files stay in place for the next run. */ +export async function reclaimAnalysisArtifacts( + directory: string, + metadata: RepoMeta, + previousGraph?: string, + log?: (message: string) => void, +): Promise { + const warn = log ?? ((message: string) => logger.warn(message)); + try { + const root = await embeddingJobDirectory(directory); + const entries = await fs.readdir(root, { withFileTypes: true }); + if (entries.some((entry) => entry.name === 'analysis-publication.json')) return; + const graphs = new Set(); + const jobs = new Set(); + const legacy = new Set(); + const keepGraph = (file: string | undefined) => { + if (file && GRAPH.test(file)) graphs.add(file); + }; + const keepJob = (id: string | undefined) => { + if (!id) return; + if (!ID.test(id)) throw new Error('Invalid referenced embedding identity'); + jobs.add(id); + }; + keepGraph(metadata.graphPath); + keepGraph(previousGraph); + if (metadata.legacyVectorSource) legacy.add(metadata.legacyVectorSource); + for (const reference of [metadata.embeddingJob, metadata.vectorGeneration]) { + if (!reference) continue; + keepJob(reference.identity); + keepJob(reference.artifactIdentity); + } + for (const owner of await readEmbeddingArtifactOwners(root)) { + keepJob(owner.artifactIdentity); + keepJob(owner.jobIdentity); + keepJob(owner.fallbackArtifact); + if (owner.phase === 'pending') keepGraph(`lbug.generation.${owner.graphGeneration}`); + if (owner.fallbackLegacy) legacy.add(owner.fallbackLegacy); + } + // Validate every reference before the first deletion, including retired + // legacy graphs that still contain the only copy of paid vectors. + for (const identity of legacy) { + const source = await readLegacyEmbeddingSource(root, identity); + keepGraph(source.sourceFile); + } + const embeddingRoot = path.join(root, 'embeddings'); + let jobEntries: string[] = []; + try { + await embeddingJobDirectory(embeddingRoot); + jobEntries = await fs.readdir(embeddingRoot); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const orphanJobs: Array<{ directory: string; receiptName: string }> = []; + for (const name of jobEntries) { + if (typeof name !== 'string' || !ID.test(name) || jobs.has(name)) continue; + const job = await embeddingJobDirectory(path.join(embeddingRoot, name)); + for (const receiptName of ['job.json', 'artifact.json', 'retained.json']) { + try { + const receipt = (await readEmbeddingJobRecord(job, receiptName)) as { + version?: number; + identity?: string; + }; + // Older retained copies have no identity in their receipt. Keep + // those unverifiable directories, as with an interrupted bootstrap. + if (receiptName === 'retained.json' && receipt.identity === undefined) break; + if (receipt.version !== 1 || receipt.identity !== name) + throw new Error('Unreferenced embedding artifact has unknown ownership'); + orphanJobs.push({ directory: job, receiptName }); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + } + const remove = async (file: string, recursive = false): Promise => { + try { + await fs.rm(file, { force: true, recursive }); + return true; + } catch (error) { + warn( + `Retained busy analysis artifact ${path.basename(file)}; cleanup will retry (${(error as Error).message}).`, + ); + return false; + } + }; + // Without a publication receipt, no attempt metadata is needed for recovery. + for (const entry of entries) + if (entry.isFile() && ATTEMPT_METADATA.test(entry.name)) + await remove(path.join(root, entry.name)); + // A busy sidecar may outlive its base graph; include it in the next sweep. + const candidates = new Set( + entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name.replace(/\.(wal|shadow)$/, '')), + ); + for (const graph of candidates) { + if (!GRAPH.test(graph) || graphs.has(graph)) continue; + if (await remove(path.join(root, graph))) + for (const suffix of ['.wal', '.shadow']) await remove(path.join(root, graph + suffix)); + } + for (const { directory: job, receiptName } of orphanJobs) { + // Keep the ownership receipt until every payload is gone. Recursive + // removal of the whole directory can delete job.json before a busy + // native file fails on Windows, making the next cleanup unverifiable. + let removed = true; + for (const name of await fs.readdir(job)) { + if (name === receiptName) continue; + if (!(await remove(path.join(job, name), true))) { + removed = false; + break; + } + } + if (removed && (await remove(path.join(job, receiptName)))) await fs.rmdir(job); + } + } catch (error) { + warn( + `Retained analysis artifacts because ownership could not be verified (${(error as Error).message}).`, + ); + } +} diff --git a/gitnexus/src/storage/analysis-slot-lock.ts b/gitnexus/src/storage/analysis-slot-lock.ts new file mode 100644 index 000000000..37c4ccccb --- /dev/null +++ b/gitnexus/src/storage/analysis-slot-lock.ts @@ -0,0 +1,53 @@ +import path from 'node:path'; +import { + acquireIndexLock, + requireExclusiveIndexLock, + type AcquireOptions, + type IndexLockHandle, +} from './index-lock.js'; +import { storeRootOfCheckoutSlot } from './shared-store.js'; +import { getGlobalDir } from './global-dir.js'; + +/** File-backed admission locks must outlive deletion of both member and store. + * Keep their empty parent directories: deleting a held gate lets a second + * process acquire a different lock at the same pathname. */ +export function sharedMemberGatePath(storeRoot: string, member: string): string { + // Store keys are unique within GITNEXUS_HOME. Keep the names as path + // components so filesystem aliases and Windows casing share the same gate. + return path.join( + getGlobalDir(), + 'locks', + 'shared-members', + path.basename(storeRoot), + path.basename(member), + ); +} + +/** Admit flat and branch writers under the member gate used by shared-store GC. + * Once admitted, their index locks protect files without serializing siblings. + * GC holds the persistent gate through member, graph, and store deletion. */ +export async function acquireAnalysisSlotLock( + directory: string, + options: AcquireOptions = {}, +): Promise { + const slot = path.resolve(directory); + const branches = path.dirname(slot); + const member = path.basename(branches) === 'branches' ? path.dirname(branches) : slot; + const storeRoot = storeRootOfCheckoutSlot(member); + if (!storeRoot) return acquireIndexLock(directory, options); + const gatePath = sharedMemberGatePath(storeRoot, member); + const gate = await acquireIndexLock(gatePath, { ...options, sweep: false }); + try { + requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${gatePath}.`); + if (slot === member) return await acquireIndexLock(directory, options); + const memberLock = await acquireIndexLock(member, { ...options, sweep: false }); + try { + requireExclusiveIndexLock(memberLock, `Cannot acquire the shared member lock at ${member}.`); + return await acquireIndexLock(directory, options); + } finally { + memberLock.release(); + } + } finally { + gate.release(); + } +} diff --git a/gitnexus/src/storage/embedding-artifact.ts b/gitnexus/src/storage/embedding-artifact.ts new file mode 100644 index 000000000..569aaaf83 --- /dev/null +++ b/gitnexus/src/storage/embedding-artifact.ts @@ -0,0 +1,304 @@ +/** Directly keyed ownership of one pending artifact per compatible input space. + * Caller holds the ordinary index lock. No model, native pool or history scan. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { + createEmbeddingJobDirectories, + embeddingJobDirectory, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, + syncEmbeddingJobDirectory, +} from './embedding-job.js'; + +export interface EmbeddingArtifactSpace { + vectorIdentity: string; + inputIdentity: string; + dimensions: number; + schema: string; +} + +export interface EmbeddingArtifactOwner { + artifactIdentity: string; + jobIdentity: string; + graphGeneration: string; + sourceIdentity: string; + manifestDigest: string; + phase: 'pending' | 'sealed'; + revalidateInputs: boolean; + /** At most one completed, read-only source for bounded group reuse. */ + fallbackArtifact?: string; + /** Alternative sole read-only fallback during first legacy migration. */ + fallbackLegacy?: string; +} + +export interface EmbeddingArtifactRecord { + version: 1; + spaceKey: string; + space: EmbeddingArtifactSpace; + owner: EmbeddingArtifactOwner; +} + +/** Read every model space before reclamation. An interrupted ownership change + * retains both sides; malformed/unreadable receipts abort the entire sweep. */ +export async function readEmbeddingArtifactOwners(root: string): Promise { + const parent = path.join(root, 'embeddings', 'spaces'); + try { + await fs.lstat(parent); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + await embeddingJobDirectory(path.dirname(parent)); + await embeddingJobDirectory(parent); + const owners: EmbeddingArtifactOwner[] = []; + for (const entry of await fs.readdir(parent, { withFileTypes: true })) { + if (!DIGEST.test(entry.name)) + throw new Error('Unexpected embedding space entry; artifacts retained'); + const directory = await embeddingJobDirectory(path.join(parent, entry.name)); + const add = (value: unknown) => { + const raw = value as EmbeddingArtifactRecord; + const validated = record(raw, raw?.space); + if (validated.spaceKey !== entry.name) throw new Error('Embedding space directory mismatch'); + owners.push(validated.owner); + }; + const current = await optionalRecord(directory, 'space.json'); + if (current !== null) add(current); + const intent = (await optionalRecord(directory, 'rebase.json')) as { + version: number; + before: unknown; + after: unknown; + } | null; + if (intent !== null) { + if (intent.version !== 1 || intent.before === undefined || !intent.after) + throw new Error('Invalid embedding rebase intent; artifacts retained'); + if (intent.before !== null) add(intent.before); + add(intent.after); + } + if (current === null && intent === null && (await fs.readdir(directory)).length) + throw new Error('Embedding space has no ownership receipt'); + } + return owners; +} + +type ValidateOwner = (owner: EmbeddingArtifactOwner) => Promise; +const DIGEST = /^[a-f0-9]{64}$/; + +export function embeddingArtifactSpaceKey(space: EmbeddingArtifactSpace): string { + if ( + !DIGEST.test(space.vectorIdentity) || + !DIGEST.test(space.inputIdentity) || + !Number.isSafeInteger(space.dimensions) || + space.dimensions < 1 || + typeof space.schema !== 'string' || + !space.schema.length || + space.schema.length > 256 + ) + throw new Error('Invalid embedding artifact space'); + return createHash('sha256') + .update( + JSON.stringify([ + 1, + space.vectorIdentity, + space.inputIdentity, + space.dimensions, + space.schema, + ]), + ) + .digest('hex'); +} + +function ownerRecord(value: unknown): EmbeddingArtifactOwner { + const owner = value as EmbeddingArtifactOwner; + if ( + !owner || + typeof owner !== 'object' || + !DIGEST.test(owner.artifactIdentity) || + !DIGEST.test(owner.jobIdentity) || + !DIGEST.test(owner.sourceIdentity) || + !DIGEST.test(owner.manifestDigest) || + !/^[a-f0-9]{32}$/.test(owner.graphGeneration) || + !['pending', 'sealed'].includes(owner.phase) || + typeof owner.revalidateInputs !== 'boolean' || + (owner.fallbackArtifact !== undefined && + (!DIGEST.test(owner.fallbackArtifact) || + owner.fallbackArtifact === owner.artifactIdentity)) || + (owner.fallbackLegacy !== undefined && + (!DIGEST.test(owner.fallbackLegacy) || owner.fallbackArtifact !== undefined)) + ) + throw new Error('Invalid embedding artifact ownership; artifacts retained'); + return { + artifactIdentity: owner.artifactIdentity, + jobIdentity: owner.jobIdentity, + graphGeneration: owner.graphGeneration, + sourceIdentity: owner.sourceIdentity, + manifestDigest: owner.manifestDigest, + phase: owner.phase, + revalidateInputs: owner.revalidateInputs, + ...(owner.fallbackArtifact === undefined ? {} : { fallbackArtifact: owner.fallbackArtifact }), + ...(owner.fallbackLegacy === undefined ? {} : { fallbackLegacy: owner.fallbackLegacy }), + }; +} + +function record(value: unknown, space: EmbeddingArtifactSpace): EmbeddingArtifactRecord { + const input = value as EmbeddingArtifactRecord; + const spaceKey = embeddingArtifactSpaceKey(space); + if ( + !input || + input.version !== 1 || + input.spaceKey !== spaceKey || + !input.space || + embeddingArtifactSpaceKey(input.space) !== spaceKey + ) + throw new Error('Embedding artifact space mismatch; artifacts retained'); + return { + version: 1, + spaceKey, + space: { + vectorIdentity: space.vectorIdentity, + inputIdentity: space.inputIdentity, + dimensions: space.dimensions, + schema: space.schema, + }, + owner: ownerRecord(input.owner), + }; +} + +async function spaceDirectory( + root: string, + space: EmbeddingArtifactSpace, + create = true, +): Promise { + let directory = await embeddingJobDirectory(root); + const parts = ['embeddings', 'spaces', embeddingArtifactSpaceKey(space)]; + if (create) return createEmbeddingJobDirectories(directory, parts); + for (const part of parts) directory = await embeddingJobDirectory(path.join(directory, part)); + return directory; +} + +async function optionalRecord(directory: string, name: string): Promise { + try { + return await readEmbeddingJobRecord(directory, name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +const same = (a: EmbeddingArtifactRecord | null, b: EmbeddingArtifactRecord | null): boolean => + JSON.stringify(a) === JSON.stringify(b); + +function validateTransition( + before: EmbeddingArtifactRecord | null, + after: EmbeddingArtifactRecord, +): void { + if (!before) return; + const a = before.owner, + b = after.owner; + if (a.artifactIdentity !== b.artifactIdentity) { + if ( + a.phase !== 'sealed' || + b.phase !== 'pending' || + b.fallbackArtifact !== a.artifactIdentity || + b.fallbackLegacy !== undefined || + b.artifactIdentity === a.fallbackArtifact + ) + throw new Error('Pending paid artifacts must be retained until sealed'); + return; + } + if (a.jobIdentity !== b.jobIdentity) { + if ( + a.phase !== 'pending' || + b.phase !== 'pending' || + !b.revalidateInputs || + a.fallbackArtifact !== b.fallbackArtifact || + a.fallbackLegacy !== b.fallbackLegacy + ) + throw new Error( + 'Rebased artifact requires current-input validation and its retained fallback', + ); + return; + } + if ( + a.graphGeneration !== b.graphGeneration || + a.sourceIdentity !== b.sourceIdentity || + a.manifestDigest !== b.manifestDigest || + (a.phase === 'sealed' && b.phase !== 'sealed') || + (b.phase === 'pending' && a.fallbackLegacy !== b.fallbackLegacy) || + (a.revalidateInputs && !b.revalidateInputs) + ) + throw new Error('Embedding artifact job identity or sealed state changed'); +} + +async function finishIntent(directory: string): Promise { + await fs.unlink(path.join(directory, 'rebase.json')); + await syncEmbeddingJobDirectory(directory); +} + +/** A lagging metadata job pointer is reconciled by the controller BEFORE it + * opens this artifact for paid work. The owned receipt names the sole writer. + * Recovery completes only the exact before/after transition, never guessing + * from an old journal cursor or choosing an artifact by directory enumeration. */ +export async function readEmbeddingArtifactSpace( + root: string, + space: EmbeddingArtifactSpace, + validateOwner: ValidateOwner, +): Promise { + let directory: string; + try { + directory = await spaceDirectory(root, space, false); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } + const currentRaw = await optionalRecord(directory, 'space.json'); + let current = currentRaw === null ? null : record(currentRaw, space); + const intent = (await optionalRecord(directory, 'rebase.json')) as { + version?: unknown; + before?: unknown; + after?: unknown; + } | null; + if (intent !== null) { + if (intent.version !== 1 || intent.before === undefined || !intent.after) + throw new Error('Invalid embedding rebase intent; artifacts retained'); + const before = intent.before === null ? null : record(intent.before, space); + const after = record(intent.after, space); + validateTransition(before, after); + if (!same(current, before) && !same(current, after)) + throw new Error('Embedding artifact owner superseded its rebase intent; artifacts retained'); + await validateOwner(after.owner); + if (!same(current, after)) await writeEmbeddingJobRecord(directory, 'space.json', after); + await finishIntent(directory); + current = after; + } + if (current) await validateOwner(current.owner); + return current; +} + +/** Manifest creation precedes this durable intent; journal recovery and native + * mutation follow it. A crash at either rename can be replayed by read above. */ +export async function adoptEmbeddingArtifact( + root: string, + space: EmbeddingArtifactSpace, + previous: EmbeddingArtifactRecord | null, + owner: EmbeddingArtifactOwner, + validateOwner: ValidateOwner, +): Promise { + const directory = await spaceDirectory(root, space); + const before = previous === null ? null : record(previous, space); + const after = record( + { version: 1, spaceKey: embeddingArtifactSpaceKey(space), space, owner }, + space, + ); + const currentRaw = await optionalRecord(directory, 'space.json'); + const current = currentRaw === null ? null : record(currentRaw, space); + if (!same(current, before) || (await optionalRecord(directory, 'rebase.json')) !== null) + throw new Error('Embedding artifact ownership changed; recover before adoption'); + validateTransition(before, after); + await validateOwner(after.owner); + if (same(before, after)) return after; + await writeEmbeddingJobRecord(directory, 'rebase.json', { version: 1, before, after }); + await writeEmbeddingJobRecord(directory, 'space.json', after); + await finishIntent(directory); + return after; +} diff --git a/gitnexus/src/storage/embedding-job-journal.ts b/gitnexus/src/storage/embedding-job-journal.ts new file mode 100644 index 000000000..cb5d74408 --- /dev/null +++ b/gitnexus/src/storage/embedding-job-journal.ts @@ -0,0 +1,503 @@ +/** Compact scheduling journal. The caller owns the checkout's exclusive job lock. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import type { + EmbeddingManifestCursor, + EmbeddingManifestReceipt, +} from '../core/embeddings/embedding-work-manifest.js'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from './embedding-job.js'; + +export interface EmbeddingUnitIntent { + start: EmbeddingManifestCursor; + next: EmbeddingManifestCursor; + nodes: number; + origin: 'manifest' | 'failed'; + /** Bounded by the unit's 64 nodes; three deterministic rejections exclude a node. */ + inputFailures?: Record; + /** Older receipts use raw node IDs. New receipts bound keys independently of ID length. */ + inputFailureKeyFormat?: 'sha256'; +} + +export interface EmbeddingJobState { + version: 1; + identity: string; + manifestDigest: string; + totalUnits: number; + totalNodes: number; + manifestBytes: number; + cursor: EmbeddingManifestCursor; + completedUnits: number; + completedNodes: number; + reusedUnits: number; + reusedNodes: number; + failedUnits: number; + failedNodes: number; + excludedNodes?: number; + chunks: number; + active: EmbeddingUnitIntent | null; + phase: 'embedding' | 'indexing' | 'ready'; +} + +export type EmbeddingFailureCause = 'provider' | 'input' | 'cancelled'; +interface FailedUnit { + version: 1; + identity: string; + manifestDigest: string; + intent: EmbeddingUnitIntent; + cause: EmbeddingFailureCause; + attempt: string; +} + +const DIGEST = /^[a-f0-9]{64}$/; +const FAILED = /^failed-(\d{16})\.json$/; +const integer = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +const filename = (unit: number): string => `failed-${String(unit).padStart(16, '0')}.json`; +const clone = (value: T): T => structuredClone(value); + +export const embeddingInputFailureKey = (nodeId: string): string => + createHash('sha256').update(nodeId).digest('hex'); + +/** Operational failures (permissions, I/O) must not be treated as lost state. */ +function damagedRecord(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + if (code) return code === 'ENOENT' || code === 'ERR_ENCODING_INVALID_ENCODED_DATA'; + return ( + error instanceof SyntaxError || + /checksum|Malformed|Invalid|UTF-8|receipt exceeds/i.test(String(error)) + ); +} + +function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): void { + if ( + !intent || + !intent.start || + !intent.next || + !integer(intent.start.unit) || + intent.start.unit >= state.totalUnits || + !integer(intent.start.offset) || + !integer(intent.next.offset) || + intent.next.offset <= intent.start.offset || + intent.next.offset > state.manifestBytes || + intent.next.unit !== intent.start.unit + 1 || + !integer(intent.nodes) || + intent.nodes < 1 || + intent.nodes > 64 || + !['manifest', 'failed'].includes(intent.origin) || + (intent.inputFailureKeyFormat !== undefined && intent.inputFailureKeyFormat !== 'sha256') + ) + throw new Error('Invalid embedding unit intent'); + if ( + intent.inputFailures !== undefined && + (!intent.inputFailures || + typeof intent.inputFailures !== 'object' || + Array.isArray(intent.inputFailures) || + Object.keys(intent.inputFailures).length > intent.nodes || + Object.entries(intent.inputFailures).some( + ([id, attempts]) => + !id.length || + id.length > 1024 * 1024 || + (intent.inputFailureKeyFormat === 'sha256' && !DIGEST.test(id)) || + !integer(attempts) || + attempts < 1 || + attempts > 3, + )) + ) + throw new Error('Invalid embedding input rejection history'); +} + +function validateState( + value: unknown, + identity: string, + manifest: EmbeddingManifestReceipt, +): asserts value is EmbeddingJobState { + const s = value as EmbeddingJobState; + if ( + !s || + s.version !== 1 || + s.identity !== identity || + s.manifestDigest !== manifest.digest || + s.totalNodes !== manifest.totalNodes || + s.totalUnits !== manifest.totalUnits || + s.manifestBytes !== manifest.bytes || + !s.cursor || + !integer(s.cursor.unit) || + s.cursor.unit > s.totalUnits || + !integer(s.cursor.offset) || + s.cursor.offset > manifest.bytes || + (s.cursor.unit === 0 && s.cursor.offset !== 0) || + (s.cursor.unit === s.totalUnits && s.cursor.offset !== manifest.bytes) || + ![ + s.completedUnits, + s.completedNodes, + s.reusedUnits, + s.reusedNodes, + s.failedUnits, + s.failedNodes, + s.chunks, + s.excludedNodes ?? 0, + ].every(integer) || + s.completedUnits + s.reusedUnits + s.failedUnits > s.totalUnits || + s.completedNodes + s.reusedNodes + s.failedNodes + (s.excludedNodes ?? 0) > s.totalNodes || + !['embedding', 'indexing', 'ready'].includes(s.phase) || + s.active === undefined + ) + throw new Error('Embedding job identity or coverage is invalid; artifacts retained'); + if (s.active) { + validateIntent(s.active, s); + const active = s.active; + if ( + active.origin === 'manifest' + ? active.start.unit !== s.cursor.unit || active.start.offset !== s.cursor.offset + : active.next.unit > s.cursor.unit + ) + throw new Error('Embedding job active cursor is inconsistent'); + } + const settled = s.completedUnits + s.reusedUnits + s.failedUnits; + if ( + settled + (s.active?.origin === 'failed' ? 1 : 0) !== s.cursor.unit || + (s.phase !== 'embedding' && + (s.active !== null || + s.failedUnits !== 0 || + settled !== s.totalUnits || + s.completedNodes + s.reusedNodes + (s.excludedNodes ?? 0) !== s.totalNodes)) + ) + throw new Error('Embedding job accounting is inconsistent'); +} + +/** No model, DB, source text, or growing pending-ID list is retained here. */ +export class EmbeddingJobJournal { + private busy = false; + private uncertain = false; + /** Failed units are attempted at most once per invocation, even if readdir sees a replacement. */ + readonly attempt = randomUUID(); + + private constructor( + readonly directory: string, + private state: EmbeddingJobState, + private readonly manifest: EmbeddingManifestReceipt, + ) {} + + static async create( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + if (!DIGEST.test(identity) || !DIGEST.test(manifest.digest)) + throw new Error('Invalid embedding job identity'); + const canonical = await embeddingJobDirectory(directory); + try { + await fs.lstat(path.join(canonical, 'job.json')); + throw new Error('Embedding job already exists'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const state: EmbeddingJobState = { + version: 1, + identity, + manifestDigest: manifest.digest, + totalUnits: manifest.totalUnits, + totalNodes: manifest.totalNodes, + manifestBytes: manifest.bytes, + cursor: { unit: 0, offset: 0 }, + completedUnits: 0, + completedNodes: 0, + reusedUnits: 0, + reusedNodes: 0, + failedUnits: 0, + failedNodes: 0, + chunks: 0, + active: null, + phase: 'embedding', + }; + validateState(state, identity, manifest); + await writeEmbeddingJobRecord(canonical, 'job.json', state); + return new EmbeddingJobJournal(canonical, state, clone(manifest)); + } + + static async open( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + if (!DIGEST.test(identity)) throw new Error('Invalid embedding job identity'); + const canonical = await embeddingJobDirectory(directory); + const state = await readEmbeddingJobRecord(canonical, 'job.json'); + validateState(state, identity, manifest); + return new EmbeddingJobJournal(canonical, state, clone(manifest)); + } + + snapshot(): EmbeddingJobState { + return clone(this.state); + } + + /** Recovery requires an independently verified manifest and vector identity. + * Preserve damaged receipts; the runner reconstructs coverage only from native + * complete groups. A valid receipt for another identity is never discarded. + * This also resumes an interruption between preserving and replacing job.json. + */ + static async recover( + directory: string, + identity: string, + manifest: EmbeddingManifestReceipt, + ): Promise { + const canonical = await embeddingJobDirectory(directory); + let value: unknown; + try { + value = await readEmbeddingJobRecord(canonical, 'job.json'); + } catch (error) { + if (!damagedRecord(error)) throw error; + } + if (value && typeof value === 'object') { + const state = value as Partial; + if ( + (state.version !== undefined && state.version !== 1) || + (state.identity && state.identity !== identity) || + (state.manifestDigest && state.manifestDigest !== manifest.digest) + ) + throw new Error('Embedding job identity mismatch; recovery refused'); + let journal: EmbeddingJobJournal | undefined; + try { + validateState(value, identity, manifest); + journal = new EmbeddingJobJournal(canonical, value, clone(manifest)); + } catch { + /* Matching identity but damaged accounting requires reconstruction. */ + } + if (journal) { + try { + await journal.verifyFailedRecords(); + return journal; + } catch (error) { + if (!damagedRecord(error)) throw error; + // Complete native node groups survive this scheduling reset. The + // runner verifies them before acknowledging without provider calls. + } + } + } + const suffix = `.preserved-${randomUUID()}`; + try { + await fs.rename(path.join(canonical, 'job.json'), path.join(canonical, `job.json${suffix}`)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await syncEmbeddingJobDirectory(canonical); + const entries = await fs.opendir(canonical); + for await (const entry of entries) { + if (FAILED.test(entry.name)) + await fs.rename( + path.join(canonical, entry.name), + path.join(canonical, entry.name + suffix), + ); + } + await syncEmbeddingJobDirectory(canonical); + return this.create(canonical, identity, manifest); + } + + private async mutate(action: (next: EmbeddingJobState) => Promise): Promise { + if (this.busy) throw new Error('Concurrent embedding journal mutation'); + if (this.uncertain) + throw new Error('Embedding journal durability uncertain; reopen before retry'); + this.busy = true; + try { + const next = clone(this.state); + await action(next); + validateState(next, next.identity, this.manifest); + // A rejected rename/fsync may already have changed durable state. Pure + // validation failures before the first write leave the journal usable. + this.uncertain = true; + await writeEmbeddingJobRecord(this.directory, 'job.json', next); + this.state = next; + this.uncertain = false; + } finally { + this.busy = false; + } + } + + async begin(intent: EmbeddingUnitIntent): Promise { + const requested = clone(intent); + await this.mutate(async (next) => { + if (next.active || next.phase !== 'embedding') + throw new Error('Embedding job already active or complete'); + validateIntent(requested, next); + if (requested.origin === 'failed') { + const record = await this.readFailed(filename(requested.start.unit)); + if ( + record.intent.start.unit !== requested.start.unit || + record.intent.start.offset !== requested.start.offset || + record.intent.next.unit !== requested.next.unit || + record.intent.next.offset !== requested.next.offset || + record.intent.nodes !== requested.nodes + ) + throw new Error('Failed embedding unit cursor mismatch'); + next.failedUnits--; + next.failedNodes -= requested.nodes; + requested.inputFailures = record.intent.inputFailures; + requested.inputFailureKeyFormat = record.intent.inputFailureKeyFormat; + } + next.active = requested; + }); + } + + /** Call only after the vector store has verified and checkpointed this unit. */ + async acknowledge(result: { reusedNodes: number; chunks: number }): Promise { + await this.mutate(async (next) => { + const active = next.active; + const excluded = Object.values(active?.inputFailures ?? {}).filter( + (attempts) => attempts >= 3, + ).length; + if ( + !active || + !integer(result.reusedNodes) || + result.reusedNodes > active.nodes - excluded || + !integer(result.chunks) + ) + throw new Error('Invalid embedding unit acknowledgement'); + // If the process dies after unlink, the still-active intent owns replay. + this.uncertain = true; + await fs.rm(path.join(this.directory, filename(active.start.unit)), { force: true }); + await syncEmbeddingJobDirectory(this.directory); + if (result.reusedNodes === active.nodes) next.reusedUnits++; + else next.completedUnits++; + next.reusedNodes += result.reusedNodes; + next.completedNodes += active.nodes - result.reusedNodes - excluded; + if (excluded) next.excludedNodes = (next.excludedNodes ?? 0) + excluded; + next.chunks += result.chunks; + if (active.origin === 'manifest') next.cursor = active.next; + next.active = null; + }); + } + + /** Input/provider failures stay durable; storage/durability failures must abort. */ + async fail(cause: EmbeddingFailureCause): Promise { + await this.mutate(async (next) => { + const active = next.active; + if (!active || !['provider', 'input', 'cancelled'].includes(cause)) + throw new Error('Invalid embedding failure'); + const record: FailedUnit = { + version: 1, + identity: next.identity, + manifestDigest: next.manifestDigest, + intent: { ...active, origin: 'failed' }, + cause, + attempt: this.attempt, + }; + this.uncertain = true; + await writeEmbeddingJobRecord(this.directory, filename(active.start.unit), record); + next.failedUnits++; + next.failedNodes += active.nodes; + if (active.origin === 'manifest') next.cursor = active.next; + next.active = null; + }); + } + + /** Persist input-only failures before discarding partial rows; return counts for these IDs. */ + async recordInputRejections(ids: readonly string[]): Promise>> { + if (!ids.length) return {}; + await this.mutate(async (next) => { + const active = next.active; + if (!active || new Set(ids).size !== ids.length || ids.length > active.nodes) + throw new Error('Invalid embedding input rejections'); + const history = Object.fromEntries( + Object.entries(active.inputFailures ?? {}).map(([key, attempts]) => [ + active.inputFailureKeyFormat === 'sha256' ? key : embeddingInputFailureKey(key), + attempts, + ]), + ); + for (const id of ids) { + if (!id.length || id.length > 1024 * 1024 || id === '__proto__') + throw new Error('Invalid rejected node identity'); + const key = embeddingInputFailureKey(id); + history[key] = Math.min(3, (history[key] ?? 0) + 1); + } + active.inputFailures = history; + active.inputFailureKeyFormat = 'sha256'; + }); + const history = this.state.active!.inputFailures!; + return Object.fromEntries(ids.map((id) => [id, history[embeddingInputFailureKey(id)]])); + } + + private async readFailed(name: string): Promise { + const record = (await readEmbeddingJobRecord(this.directory, name)) as FailedUnit; + if ( + record && + typeof record === 'object' && + ((record.version !== undefined && record.version !== 1) || + (record.identity && record.identity !== this.state.identity) || + (record.manifestDigest && record.manifestDigest !== this.state.manifestDigest)) + ) + throw new Error('Embedding failure identity mismatch; recovery refused'); + if ( + !record || + record.version !== 1 || + record.identity !== this.state.identity || + record.manifestDigest !== this.state.manifestDigest || + !['provider', 'input', 'cancelled'].includes(record.cause) || + typeof record.attempt !== 'string' + ) + throw new Error('Invalid failed embedding unit'); + validateIntent(record.intent, this.state); + if (record.intent.origin !== 'failed' || name !== filename(record.intent.start.unit)) + throw new Error('Failed embedding unit identity mismatch'); + return record; + } + + /** Constant memory; failed records are the only growing scheduling data. */ + private async verifyFailedRecords(): Promise { + let units = 0; + let nodes = 0; + const entries = await fs.opendir(this.directory); + for await (const entry of entries) { + if (!FAILED.test(entry.name)) continue; + const record = await this.readFailed(entry.name); + // An active retry owns this unit even if its failure file was already + // removed, or a crash left the previous failed record beside it. + if (this.state.active && entry.name === filename(this.state.active.start.unit)) continue; + units++; + nodes += record.intent.nodes; + if ( + record.intent.next.unit > this.state.cursor.unit || + units > this.state.failedUnits || + nodes > this.state.failedNodes + ) + throw new Error('Invalid failed embedding accounting'); + } + if (units !== this.state.failedUnits || nodes !== this.state.failedNodes) + throw new Error('Invalid failed embedding accounting'); + } + + async *failed(): AsyncGenerator { + const directory = await fs.opendir(this.directory); + for await (const entry of directory) { + if (!FAILED.test(entry.name)) continue; + const record = await this.readFailed(entry.name); + if ( + record.attempt === this.attempt || + record.intent.start.unit === this.state.active?.start.unit + ) + continue; + if (record.intent.next.unit > this.state.cursor.unit) + throw new Error('Orphan failed unit has no active intent'); + yield clone(record.intent); + } + } + + async indexing(): Promise { + await this.mutate(async (next) => { + next.phase = 'indexing'; + }); + } + + /** The completed vector artifact must be closed and published before this call. */ + async ready(): Promise { + await this.mutate(async (next) => { + if (next.phase !== 'indexing') throw new Error('Embedding index has not been finalized'); + next.phase = 'ready'; + }); + } +} diff --git a/gitnexus/src/storage/embedding-job-progress.ts b/gitnexus/src/storage/embedding-job-progress.ts new file mode 100644 index 000000000..0716c0444 --- /dev/null +++ b/gitnexus/src/storage/embedding-job-progress.ts @@ -0,0 +1,74 @@ +/** Observation only: bounded journal read, no source scan, native DB, or model. */ +import path from 'node:path'; +import { + embeddingProgressFromState, + type EmbeddingJobProgress, +} from '../core/analysis-progress.js'; +import type { RepoMeta } from './repo-meta.js'; +import type { EmbeddingJobState } from './embedding-job-journal.js'; +import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js'; +import { embeddingGenerationDirectory, selectVectorGeneration } from './vector-generation.js'; +import { getExactScanLimit } from '../core/platform/capabilities.js'; + +export async function readEmbeddingJobProgress( + metadataDirectory: string, + metadata: RepoMeta | null | undefined, +): Promise { + const reference = metadata?.embeddingJob; + if (!reference) { + if ( + !metadata?.vectorGeneration?.preservedForGraph && + !metadata?.vectorGeneration?.excludedNodes + ) + return undefined; + if (metadata.graphPath !== undefined && typeof metadata.graphPath !== 'string') + return { phase: 'unverified' }; + const selected = selectVectorGeneration( + metadataDirectory, + metadata, + path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'), + getExactScanLimit(), + ); + // These are historical chunks, not completed/reused work in a current + // manifest. Unknown selection totals stay absent, never zero or complete. + return selected.kind === 'ready' + ? { + phase: selected.coverage === 'preserved' ? 'preserved' : 'ready', + chunks: selected.generation.chunks, + ...(selected.generation.excludedNodes + ? { excludedNodes: selected.generation.excludedNodes } + : {}), + } + : undefined; + } + try { + if ( + reference.version !== 1 || + metadata?.graphGeneration?.version !== 1 || + !/^[a-f0-9]{32}$/.test(reference.graphGeneration) || + reference.graphGeneration !== metadata.graphGeneration.id + ) + throw new Error('Job graph mismatch'); + const directory = embeddingGenerationDirectory(metadataDirectory, reference.identity); + // Validate every job-owned directory before following the path. In + // particular, a symlinked embeddings parent must not redirect this read. + await embeddingJobDirectory(metadataDirectory); + await embeddingJobDirectory(path.dirname(directory)); + const state = (await readEmbeddingJobRecord(directory, 'job.json')) as EmbeddingJobState; + if ( + !state || + state.version !== 1 || + state.identity !== reference.identity || + !/^[a-f0-9]{64}$/.test(state.manifestDigest) + ) + throw new Error('Job identity mismatch'); + const progress = embeddingProgressFromState(state); + // A journal alone cannot publish a vector capability. If the metadata + // still points at pending work, even a ready journal awaits publication. + if (progress.phase === 'ready') progress.phase = 'indexing'; + return progress; + } catch { + // Missing/corrupt/unreadable work is not a zero-work or completed job. + return { phase: 'unverified' }; + } +} diff --git a/gitnexus/src/storage/embedding-job.ts b/gitnexus/src/storage/embedding-job.ts new file mode 100644 index 000000000..13063c4b5 --- /dev/null +++ b/gitnexus/src/storage/embedding-job.ts @@ -0,0 +1,164 @@ +/** Filesystem-only embedding receipts; importing this never loads a model or DB. */ +import { createHash, randomUUID } from 'node:crypto'; +import { constants, type Stats } from 'node:fs'; +import fs, { type FileHandle } from 'node:fs/promises'; +import path from 'node:path'; + +const MAX_RECORD_BYTES = 32 * 1024; +const RECORD_NAMES = new Set([ + 'manifest.json', + 'job.json', + 'space.json', + 'rebase.json', + 'legacy.json', + 'retained.json', + 'artifact.json', +]); +const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/; + +/** Pin the file before checking it; a replaced pathname cannot redirect reads. */ +export async function openEmbeddingJobFile( + file: string, +): Promise<{ handle: FileHandle; stat: Stats }> { + let handle: FileHandle; + try { + handle = await fs.open( + file, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ELOOP') + throw new Error('Embedding artifact must be a regular file', { cause: error }); + throw error; + } + try { + const stat = await handle.stat(); + if (!stat.isFile()) throw new Error('Embedding artifact must be a regular file'); + // Windows lacks O_NOFOLLOW. Validate the directory entry against the + // pinned descriptor before reading any bytes; later path swaps are harmless. + if (!constants.O_NOFOLLOW) { + const entry = await fs.lstat(file); + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Embedding artifact must be a regular file'); + if (entry.dev !== stat.dev || entry.ino !== stat.ino) + throw new Error('Embedding artifact changed during open'); + } + return { handle, stat }; + } catch (error) { + await handle.close(); + throw error; + } +} + +export async function embeddingJobDirectory(directory: string): Promise { + const stat = await fs.lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Embedding job directory must be an owned directory'); + const canonical = await fs.realpath(directory); + const resolved = await fs.lstat(canonical); + if ( + !resolved.isDirectory() || + resolved.isSymbolicLink() || + resolved.dev !== stat.dev || + resolved.ino !== stat.ino + ) + throw new Error('Embedding job directory changed during resolution'); + return canonical; +} + +/** Root is already canonical and validated; validate each child before traversing it. */ +export async function createEmbeddingJobDirectories( + root: string, + components: readonly string[], +): Promise { + let directory = root; + for (const component of components) { + const next = path.join(directory, component); + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + +/** Flush directory entries where the platform supports opening directories. */ +export async function syncEmbeddingJobDirectory(directory: string): Promise { + if (process.platform === 'win32') return; + const handle = await fs.open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0)); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function writeEmbeddingJobRecord( + directory: string, + name: string, + payload: unknown, +): Promise { + if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name)) + throw new Error('Invalid embedding receipt name'); + const canonical = await embeddingJobDirectory(directory); + const serialized = JSON.stringify(payload); + if (serialized === undefined) throw new Error('Embedding receipt payload is not JSON'); + const checksum = createHash('sha256').update(serialized).digest('hex'); + const bytes = Buffer.from(JSON.stringify({ checksum, payload }), 'utf8'); + if (bytes.byteLength > MAX_RECORD_BYTES) + throw new Error('Embedding receipt exceeds its fixed size limit'); + const temporary = path.join(canonical, `.${name}.${randomUUID()}.tmp`); + const handle = await fs.open(temporary, 'wx', 0o600); + try { + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, path.join(canonical, name)); + await syncEmbeddingJobDirectory(canonical); + } finally { + await fs.rm(temporary, { force: true }); + } +} + +/** Bounded read; malformed or untrusted receipts never become an empty job. */ +export async function readEmbeddingJobRecord(directory: string, name: string): Promise { + if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name)) + throw new Error('Invalid embedding receipt name'); + const canonical = await embeddingJobDirectory(directory); + const target = path.join(canonical, name); + const { handle } = await openEmbeddingJobFile(target); + try { + const buffer = Buffer.alloc(MAX_RECORD_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > MAX_RECORD_BYTES) + throw new Error('Embedding receipt exceeds its fixed size limit'); + const parsed: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)), + ); + if ( + !parsed || + typeof parsed !== 'object' || + !('checksum' in parsed) || + !('payload' in parsed) + ) { + throw new Error('Malformed embedding receipt'); + } + const checksum = createHash('sha256').update(JSON.stringify(parsed.payload)).digest('hex'); + if (parsed.checksum !== checksum) + throw new Error('Embedding receipt checksum mismatch; artifacts retained'); + return parsed.payload; + } finally { + await handle.close(); + } +} diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 72f59f618..c3c9317bd 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -1,5 +1,5 @@ import { execFileSync, execSync, spawnSync } from 'child_process'; -import { statSync, existsSync } from 'fs'; +import { statSync, existsSync, lstatSync, realpathSync } from 'fs'; import path from 'path'; import os from 'os'; import { logger } from '../core/logger.js'; @@ -585,6 +585,21 @@ export const getCoreExcludesFilePath = (fromPath: string): string => { */ export const resolveRepoIdentityRoot = (fromPath: string): string => { const resolved = path.resolve(fromPath); + // An ordinary checkout is its own identity root. Let Git interpret worktrees, + // symlinks, common-directory indirection and environment overrides as before. + try { + const gitDir = path.join(resolved, '.git'); + if ( + process.env.GIT_DIR === undefined && + process.env.GIT_COMMON_DIR === undefined && + lstatSync(gitDir).isDirectory() && + !existsSync(path.join(gitDir, 'commondir')) && + realpathSync.native(resolved) === resolved + ) + return resolved; + } catch { + // Missing or unreadable paths retain the existing Git/fallback behavior. + } const canonical = getCanonicalRepoRoot(resolved); if (!canonical) return resolved; // non-git → use as-is if (canonical === resolved) return canonical; // canonical checkout diff --git a/gitnexus/src/storage/index-capabilities.ts b/gitnexus/src/storage/index-capabilities.ts new file mode 100644 index 000000000..82b310325 --- /dev/null +++ b/gitnexus/src/storage/index-capabilities.ts @@ -0,0 +1,66 @@ +/** Published capability receipts, gated by current routing and artifact ownership. + * Runtime query probes may further degrade these facts; status never loads a DB. */ +import type { RepoMeta } from './repo-meta.js'; +import { resolveGraphPath } from './shared-store.js'; +import { assertOwnedVectorGeneration, selectVectorGeneration } from './vector-generation.js'; +import { getExactScanLimit } from '../core/platform/capabilities.js'; + +export async function readIndexCapabilities( + directory: string, + metadata: RepoMeta | null | undefined, +): Promise { + if ( + !metadata?.graphGeneration && + !metadata?.vectorGeneration && + !metadata?.embeddingJob && + !metadata?.incrementalInProgress && + !metadata?.graphWriteCollapsed + ) + return metadata?.capabilities; + const exactScanLimit = getExactScanLimit(); + let vectorSearch: NonNullable['vectorSearch']; + try { + const selection = selectVectorGeneration( + directory, + metadata, + resolveGraphPath(directory, metadata), + exactScanLimit, + ); + if (selection.kind === 'ready') { + await assertOwnedVectorGeneration( + directory, + selection.generation.artifactIdentity ?? selection.generation.identity, + ); + vectorSearch = { + provider: selection.generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan', + status: selection.generation.indexKind === 'hnsw' ? 'vector-index' : 'exact-scan', + exactScanLimit, + coverage: selection.coverage ?? 'current', + sourceGraphGeneration: selection.generation.graphGeneration, + ...(selection.generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}), + }; + } else if (selection.kind === 'legacy' && metadata?.capabilities?.vectorSearch) { + vectorSearch = { ...metadata.capabilities.vectorSearch, coverage: 'preserved' }; + } else { + vectorSearch = { + provider: 'ladybugdb-vector', + status: 'unavailable', + exactScanLimit, + reason: + selection.kind === 'unavailable' ? selection.reason : 'vector-generation-unverified', + }; + } + } catch { + vectorSearch = { + provider: 'ladybugdb-vector', + status: 'unavailable', + exactScanLimit, + reason: 'vector-artifact-unavailable', + }; + } + return { + graph: metadata?.capabilities?.graph ?? { provider: 'ladybugdb', status: 'unavailable' }, + fts: metadata?.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' }, + vectorSearch, + }; +} diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index 5d3b6cc47..6a146fa4b 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -226,7 +226,7 @@ const unverifiedGuardError = (guardPath: string): Error => const HOSTNAME = os.hostname(); /** Linux: field 22 of /proc//stat (starttime). null elsewhere / on error. */ -const readProcStartTime = (pid: number): string | null => { +export const readProcStartTime = (pid: number): string | null => { if (process.platform !== 'linux') return null; try { const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); @@ -441,9 +441,19 @@ const deniedCreateHandle = ( * `branches//` sub-slot (which owns its own lock + sweep). Non-recursive. */ export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => void): void => { + // A graph worker releases this lock before its supervisor reacquires it. + // Presence (even malformed or unreadable) retains the attempt until the + // publication coordinator validates it; lock ownership alone cannot prove + // that these stages are orphaned. + try { + lstatSync(path.join(lockDir, 'analysis-publication.json')); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return; + } // Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.`, `lbug.staging..wal`, … // Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`. - const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/; + const stagingRe = /^lbug\.(?:generation\.[a-f0-9]{32}\.)?(staging\..+|new(\..+)?)$/; const retained = new Set(readEmbeddingRecovery(lockDir)?.familyFiles ?? []); let removed = 0; let entries: string[]; diff --git a/gitnexus/src/storage/legacy-embedding-source.ts b/gitnexus/src/storage/legacy-embedding-source.ts new file mode 100644 index 000000000..f4f0c1574 --- /dev/null +++ b/gitnexus/src/storage/legacy-embedding-source.ts @@ -0,0 +1,295 @@ +/** Filesystem-only provenance for one retired, immutable legacy database. + * The caller holds the slot lock and retires/checkpoints the source before + * recording it. No current configuration is used to fill historical fields. */ +import { createHash, randomUUID } from 'node:crypto'; +import type { BigIntStats } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { EmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; +import { assertValidGraphNodeId } from '../core/graph/node-identity.js'; +import { + embeddingJobDirectory, + openEmbeddingJobFile, + readEmbeddingJobRecord, + writeEmbeddingJobRecord, + syncEmbeddingJobDirectory, +} from './embedding-job.js'; + +const DIGEST = /^[a-f0-9]{64}$/; +const SOURCE = + /^lbug\.generation\.[a-f0-9]{32}$|^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/; +const IO_BYTES = 64 * 1024; +const MAX_ID_BYTES = 1024 * 1024; +const hash = (value: unknown): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); +const count = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +export interface LegacyEmbeddingSourceRecord { + version: 1; + sourceFile: string; + schemaFingerprint: string; + recordedIdentity?: Partial; + recordedInputIdentity?: string; + /** Descriptor attributes detect replacement/mutation. Ownership and provider + * provenance come from the retired slot and record, never these attributes. */ + file: { dev: string; ino: string; size: string; mtimeNs: string; ctimeNs: string }; + excluded: { digest: string; bytes: number; nodes: number }; +} +export interface ResolvedLegacyEmbeddingSource extends LegacyEmbeddingSourceRecord { + identity: string; + path: string; + excludedIds: () => AsyncIterable; +} + +function identityFields( + value: Partial | undefined, +): Partial | undefined { + if (value === undefined) return undefined; + if ( + !value || + typeof value !== 'object' || + (value.provider !== undefined && !/^(local|http:[a-f0-9]{64})$/.test(value.provider)) || + (value.model !== undefined && + (typeof value.model !== 'string' || !value.model || value.model.length > 2048)) || + (value.dimensions !== undefined && (!count(value.dimensions) || !value.dimensions)) + ) + throw new Error('Invalid recorded legacy embedding identity'); + return { + ...(value.provider === undefined ? {} : { provider: value.provider }), + ...(value.model === undefined ? {} : { model: value.model }), + ...(value.dimensions === undefined ? {} : { dimensions: value.dimensions }), + }; +} +function record(value: unknown): LegacyEmbeddingSourceRecord { + const r = value as LegacyEmbeddingSourceRecord; + if ( + !r || + r.version !== 1 || + typeof r.sourceFile !== 'string' || + !SOURCE.test(r.sourceFile) || + typeof r.schemaFingerprint !== 'string' || + !r.schemaFingerprint || + r.schemaFingerprint.length > 2048 || + (r.recordedInputIdentity !== undefined && !DIGEST.test(r.recordedInputIdentity)) || + !r.file || + ![r.file.dev, r.file.ino, r.file.size, r.file.mtimeNs, r.file.ctimeNs].every( + (v) => typeof v === 'string' && /^\d+$/.test(v), + ) || + !r.excluded || + !DIGEST.test(r.excluded.digest) || + !count(r.excluded.bytes) || + !count(r.excluded.nodes) + ) + throw new Error('Invalid legacy source receipt; source retained'); + const recordedIdentity = identityFields(r.recordedIdentity); + return { + version: 1, + sourceFile: r.sourceFile, + schemaFingerprint: r.schemaFingerprint, + ...(recordedIdentity === undefined ? {} : { recordedIdentity }), + ...(r.recordedInputIdentity === undefined + ? {} + : { recordedInputIdentity: r.recordedInputIdentity }), + file: { + dev: r.file.dev, + ino: r.file.ino, + size: r.file.size, + mtimeNs: r.file.mtimeNs, + ctimeNs: r.file.ctimeNs, + }, + excluded: { digest: r.excluded.digest, bytes: r.excluded.bytes, nodes: r.excluded.nodes }, + }; +} +function fileIdentity(stat: BigIntStats): LegacyEmbeddingSourceRecord['file'] { + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Legacy source must be an owned regular file'); + return { + dev: String(stat.dev), + ino: String(stat.ino), + size: String(stat.size), + mtimeNs: String(stat.mtimeNs), + ctimeNs: String(stat.ctimeNs), + }; +} +async function sourceFile(root: string, r: LegacyEmbeddingSourceRecord): Promise { + const file = path.join(root, r.sourceFile); + if ( + JSON.stringify(fileIdentity(await fs.lstat(file, { bigint: true }))) !== JSON.stringify(r.file) + ) + throw new Error('Legacy source changed after retirement; artifacts retained'); + // A read-only migration cannot recover an interrupted native write. Its + // graph-phase owner must checkpoint/recover it before writing this receipt. + for (const suffix of ['.wal', '.shadow']) { + try { + const stat = await fs.lstat(file + suffix); + if (!stat.isFile() || stat.size) + throw new Error('Legacy source needs native recovery before reuse'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + return file; +} +async function legacyRoot(root: string, create: boolean): Promise { + let directory = await embeddingJobDirectory(root); + for (const part of ['embeddings', 'legacy']) { + const next = path.join(directory, part); + if (create) { + try { + await fs.mkdir(next, { mode: 0o700 }); + await syncEmbeddingJobDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + } + directory = await embeddingJobDirectory(next); + } + return directory; +} + +/** Streams checkpoint exclusions once. The small receipt is published last, + * after the spool is durable. The native source is never copied or modified. */ +export async function recordLegacyEmbeddingSource( + root: string, + input: Pick< + LegacyEmbeddingSourceRecord, + 'sourceFile' | 'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity' + >, + excluded: AsyncIterable | Iterable, + signal?: AbortSignal, +): Promise { + root = await embeddingJobDirectory(root); + if (!SOURCE.test(input.sourceFile)) + throw new Error('Legacy source must be a retired owned graph'); + const file = fileIdentity(await fs.lstat(path.join(root, input.sourceFile), { bigint: true })); + const parent = await legacyRoot(root, true); + const temporary = path.join(parent, `.prepare-${randomUUID()}`); + await fs.mkdir(temporary, { mode: 0o700 }); + try { + const handle = await fs.open(path.join(temporary, 'excluded.jsonl'), 'wx', 0o600); + const checksum = createHash('sha256'); + let bytes = 0, + nodes = 0, + pending = '', + pendingBytes = 0; + try { + for await (const id of excluded) { + signal?.throwIfAborted(); + assertValidGraphNodeId(id, 'Legacy checkpoint'); + const line = JSON.stringify(id) + '\n'; + const length = Buffer.byteLength(line); + if (length > MAX_ID_BYTES || !Number.isSafeInteger(bytes + length)) + throw new Error('Legacy exclusion exceeds its bounded size'); + checksum.update(line); + bytes += length; + nodes++; + pending += line; + pendingBytes += length; + if (pendingBytes >= IO_BYTES) { + await handle.writeFile(pending); + pending = ''; + pendingBytes = 0; + } + } + if (pendingBytes) await handle.writeFile(pending); + await handle.sync(); + } finally { + await handle.close(); + } + signal?.throwIfAborted(); + const receipt = record({ + version: 1, + ...input, + file, + excluded: { digest: checksum.digest('hex'), bytes, nodes }, + }); + await sourceFile(root, receipt); + const identity = hash(receipt); + await writeEmbeddingJobRecord(temporary, 'legacy.json', receipt); + const destination = path.join(parent, identity); + try { + await fs.rename(temporary, destination); + } catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) + throw error; + // An identical request is idempotent, but a corrupt prior spool is not + // silently replaced. Fully validate it before trusting the same name. + const prior = await readLegacyEmbeddingSource(root, identity); + for await (const _id of prior.excludedIds()) signal?.throwIfAborted(); + } + await syncEmbeddingJobDirectory(parent); + return identity; + } finally { + await fs.rm(temporary, { recursive: true, force: true }); + } +} + +async function* exclusions( + directory: string, + expected: LegacyEmbeddingSourceRecord['excluded'], +): AsyncGenerator { + const file = path.join(directory, 'excluded.jsonl'); + const { handle, stat } = await openEmbeddingJobFile(file); + try { + if (stat.size !== expected.bytes) throw new Error('Legacy exclusions size mismatch'); + const buffer = Buffer.alloc(IO_BYTES), + checksum = createHash('sha256'); + let pending = Buffer.alloc(0), + bytes = 0, + nodes = 0; + for (;;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, bytes); + if (!bytesRead) break; + bytes += bytesRead; + checksum.update(buffer.subarray(0, bytesRead)); + const data = pending.length + ? Buffer.concat([pending, buffer.subarray(0, bytesRead)]) + : buffer.subarray(0, bytesRead); + let offset = 0; + for (;;) { + const end = data.indexOf(10, offset); + if (end < 0) break; + if (end - offset + 1 > MAX_ID_BYTES) + throw new Error('Legacy exclusion exceeds its bounded size'); + const id: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)), + ); + assertValidGraphNodeId(id, 'Legacy checkpoint'); + nodes++; + yield id; + offset = end + 1; + } + pending = Buffer.from(data.subarray(offset)); + if (pending.length >= MAX_ID_BYTES || bytes > expected.bytes) + throw new Error('Legacy exclusions exceed their bounds'); + } + if ( + pending.length || + bytes !== expected.bytes || + nodes !== expected.nodes || + checksum.digest('hex') !== expected.digest + ) + throw new Error('Legacy exclusions checksum mismatch; source retained'); + } finally { + await handle.close(); + } +} + +export async function readLegacyEmbeddingSource( + root: string, + identity: string, +): Promise { + if (!DIGEST.test(identity)) throw new Error('Invalid legacy source identity'); + root = await embeddingJobDirectory(root); + const directory = await embeddingJobDirectory(path.join(await legacyRoot(root, false), identity)); + const receipt = record(await readEmbeddingJobRecord(directory, 'legacy.json')); + if (hash(receipt) !== identity) + throw new Error('Legacy source identity mismatch; source retained'); + return { + ...receipt, + identity, + path: await sourceFile(root, receipt), + excludedIds: () => exclusions(directory, receipt.excluded), + }; +} diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index f2f0086e5..96ea04498 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -50,7 +50,6 @@ import { type AnalyzerRunnerIdentity, type RepoMeta, } from './repo-meta.js'; -import { LBUG_DIRECTORY } from './storage-constants.js'; import { resolveGraphPath } from './shared-store.js'; import { defaultStoragePath, @@ -243,9 +242,9 @@ export const getStoragePaths = ( const baseDir = branch ? path.join(storagePath, BRANCHES_DIR, branchSlug(branch)) : storagePath; return { storagePath, - // Branch slots are always private; a flat shared-store slot may read a - // commit graph (#3352). - lbugPath: branch ? path.join(baseDir, LBUG_DIRECTORY) : resolveGraphPath(storagePath), + // Private generations also work in branch slots. Only a flat shared-store + // slot may resolve to an immutable commit graph (#3352). + lbugPath: resolveGraphPath(baseDir), metaPath: path.join(baseDir, INDEX_METADATA_FILE), // Branch-specific metadata file }; }; @@ -945,6 +944,7 @@ export class RegistryNameCollisionError extends Error { const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): boolean => { const resolved = path.resolve(entry.path); if (entry.name === path.basename(resolved)) return false; + if (inferredName && entry.name === inferredName) return false; // Canonical-root-derived names are not user aliases either (#1259): // a worktree registered under the canonical repo's basename // (e.g. `{name: 'repo', path: '/repo/wt-feature'}`) must re-register @@ -956,7 +956,6 @@ const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): bool // — analyze canonical, then analyze worktree, then re-analyze worktree // — would break on the third call. if (entry.name === path.basename(resolveRepoIdentityRoot(resolved))) return false; - if (inferredName && entry.name === inferredName) return false; return true; }; diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 40bc80494..8c095aff4 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -114,11 +114,51 @@ export interface RepoMeta { /** Complete index directory selected for this successful analysis. */ storagePath?: string; /** - * Shared-store checkouts only (#3352): the graph this slot reads when it is - * not `/lbug` — a commit graph under the same store. Resolved - * and validated by `resolveGraphPath`. + * The graph this slot reads: an owned private generation basename, or an + * absolute shared commit graph under the same store (#3352). Resolved and + * validated by `resolveGraphPath`. Absent means legacy `/lbug`. */ graphPath?: string; + /** A verified logical generation, published independently of optional vectors. + * Private incremental writes invalidate this receipt before mutating its file. */ + graphGeneration?: { + version: 1; + id: string; + /** Digest of covered content, including uncommitted source changes. */ + coveredContent: string; + }; + /** Checkout-owned durable work. Progress lives in its bounded journal. */ + embeddingJob?: { + version: 1; + identity: string; + graphGeneration: string; + /** Mutable artifact ownership is separate from this graph-bound job. */ + artifactIdentity?: string; + spaceKey?: string; + }; + /** Direct receipt for a retained legacy source; missing historical fields + * remain unknown. The source is immutable and separate from graphPath. */ + legacyVectorSource?: string; + /** Published only after complete coverage, native durability and index validation. */ + vectorGeneration?: { + version: 1; + identity: string; + /** Absent in older separate stores, whose artifact identity equaled the job. */ + artifactIdentity?: string; + graphGeneration: string; + /** Explicit permission to serve this completed historical artifact against + * a newer graph. graphGeneration remains its original coverage receipt. */ + preservedForGraph?: string; + vectorIdentity: string; + inputIdentity: string; + dimensions: number; + nodes: number; + chunks: number; + /** Deterministic input failures exhausted three attempts in this job. */ + excludedNodes?: number; + exclusionReason?: 'input-rejected'; + indexKind: 'hnsw' | 'exact'; + }; lastCommit: string; indexedAt: string; /** Missing on legacy metadata means the upstream-compatible `full` profile. */ @@ -248,6 +288,9 @@ export interface RepoMeta { status: 'vector-index' | 'exact-scan' | 'unavailable'; exactScanLimit: number; reason?: string; + /** Native availability and coverage of the current selection differ. */ + coverage?: 'current' | 'preserved' | 'unknown'; + sourceGraphGeneration?: string; }; }; /** @@ -459,6 +502,8 @@ export interface RepoMeta { * to a known-good index. */ incrementalInProgress?: { + /** Unique ownership fence for a prepared in-place graph publication. */ + attemptId?: string; /** When the run started (epoch ms). */ startedAt: number; /** Last dirty-flag refresh (epoch ms). */ diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index 081441cc7..bc4d5b51d 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -13,6 +13,7 @@ import { existsSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { acquireIndexLock, requireExclusiveIndexLock, type IndexLockHandle } from './index-lock.js'; +import { sharedMemberGatePath } from './analysis-slot-lock.js'; import { canonicalizePath, findRegistryEntryByRepoPath, @@ -25,12 +26,7 @@ import { storeRootOfCheckoutSlot, type SharedStoreLayout, } from './shared-store.js'; -import { - GITNEXUS_DIR, - INDEX_METADATA_FILE, - LBUG_DIRECTORY, - LEGACY_METADATA_FILE, -} from './storage-constants.js'; +import { GITNEXUS_DIR, INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './storage-constants.js'; type StoreRoot = Pick; @@ -105,6 +101,50 @@ const loadMetaStrict = async (slot: string): Promise => { return null; }; +/** Each branch owns its own publication and vector work under the member's + * ordinary slot lock. Never follow a symlink while deciding what to collect. */ +async function memberMetadataDirectories(slot: string): Promise { + const directories = [slot]; + const branches = path.join(slot, 'branches'); + try { + const stat = await fs.lstat(branches); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Cannot collect a shared member with an unverified branches directory'); + for (const entry of await fs.readdir(branches, { withFileTypes: true })) { + if (entry.isSymbolicLink()) + throw new Error('Cannot collect a shared member with a symlinked branch'); + if (entry.isDirectory() && !entry.name.startsWith('.')) + directories.push(path.join(branches, entry.name)); + } + } catch (error) { + if (!isMissingFilesystemError(error)) throw error; + } + return directories; +} + +/** The writer releases its lock before an exit-only supervisor reacquires it. + * Its durable intent still owns the files throughout that gap and a restart. + * A malformed receipt is retained evidence, never permission to delete it. */ +async function hasRetainedAnalysis(slot: string): Promise { + for (const directory of await memberMetadataDirectories(slot)) { + try { + await fs.lstat(path.join(directory, 'analysis-publication.json')); + return true; + } catch (error) { + if (!isMissingFilesystemError(error)) throw error; + } + const metadata = await loadMetaStrict(directory); + if ( + metadata && + (Object.hasOwn(metadata, 'embeddingJob') || + Object.hasOwn(metadata, 'embeddingCheckpoint') || + Object.hasOwn(metadata, 'legacyVectorSource')) + ) + return true; + } + return false; +} + /** * Member slots that no registry entry uses any more: the checkout directory is * gone, or its entry moved elsewhere (`--no-share`, sharing turned off). The @@ -121,7 +161,7 @@ const orphanMembers = async (slots: string[]): Promise> => { const meta = await loadMeta(slot); if (!meta?.repoPath) continue; if (!existsSync(meta.repoPath)) { - orphans.add(slot); + if (!(await hasRetainedAnalysis(slot))) orphans.add(slot); continue; } if (!entries) continue; @@ -130,7 +170,7 @@ const orphanMembers = async (slots: string[]): Promise> => { !entry || !registryPathEquals(canonicalizePath(entry.storagePath), canonicalizePath(slot)) ) { - orphans.add(slot); + if (!(await hasRetainedAnalysis(slot))) orphans.add(slot); } } return orphans; @@ -158,87 +198,128 @@ export const reclaimSharedStoreLocked = async ( const checkoutsDir = path.join(storeRoot, 'checkouts'); const commitsDir = path.join(storeRoot, 'commits'); const referenced = new Set(); - let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name)); - if (opts.gc) { - const orphans = await orphanMembers(slots); - for (const slot of [...orphans]) { - // An analyze holds its slot's index lock until it has registered the - // checkout, so a slot that is seeded but not yet registered is busy, - // not orphaned. Judge (and, unless previewing, delete) only a slot - // whose lock is free, so the preview matches what --force would do. - let lock: IndexLockHandle; - try { - // A preview deletes nothing, not even the staging files the lock sweeps. - lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: !opts.dryRun }); - } catch { - orphans.delete(slot); - continue; - } - try { - if (lock.lockFree || !(await orphanMembers([slot])).has(slot)) { + const heldMemberGates: IndexLockHandle[] = []; + try { + let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name)); + if (opts.gc) { + const orphans = await orphanMembers(slots); + for (const slot of [...orphans]) { + // An analyze holds its slot's index lock until it has registered the + // checkout, so a slot that is seeded but not yet registered is busy, + // not orphaned. Judge (and, unless previewing, delete) only a slot + // whose lock is free, so the preview matches what --force would do. + let lock: IndexLockHandle; + let gate: IndexLockHandle | undefined; + try { + gate = await acquireIndexLock(sharedMemberGatePath(storeRoot, slot), { + timeoutMs: 1, + sweep: false, + }); + requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${slot}.`); + // Recheck durable work under ownership before deleting anything. In + // particular, acquiring the lock must not sweep a just-prepared graph. + lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: false }); + } catch { + gate?.release(); orphans.delete(slot); continue; } - if (!opts.dryRun) { - try { - await fs.rm(slot, { recursive: true, force: true }); - } catch { - // Like an undeletable graph below: keep it for the next collection - // rather than abort every store after this one. It stays a member, - // so whatever graph it still names is kept too. + const branchLocks: IndexLockHandle[] = []; + let dropped = false; + let deleting = false; + try { + let busy = lock.lockFree; + if (!busy) { + for (const branch of (await memberMetadataDirectories(slot)).slice(1)) { + try { + const branchLock = await acquireIndexLock(branch, { timeoutMs: 1, sweep: false }); + branchLocks.push(branchLock); + if (branchLock.lockFree) { + busy = true; + break; + } + } catch { + busy = true; + break; + } + } + } + if (busy || !(await orphanMembers([slot])).has(slot)) { orphans.delete(slot); - result.keptMembers.push(slot); continue; } + if (!opts.dryRun) { + try { + deleting = true; + await fs.rm(slot, { recursive: true, force: true }); + } catch { + // Like an undeletable graph below: keep it for the next collection + // rather than abort every store after this one. It stays a member, + // so whatever graph it still names is kept too. + orphans.delete(slot); + result.keptMembers.push(slot); + continue; + } + } + result.droppedMembers.push(slot); + dropped = true; + } finally { + for (const branchLock of branchLocks) branchLock.release(); + lock.release(); + // A deleted member must not be recreated until all graph and store + // deletion finishes. This gate is outside the recursively removed tree. + if (dropped || deleting) heldMemberGates.push(gate); + else gate.release(); } - result.droppedMembers.push(slot); - } finally { - lock.release(); + } + slots = slots.filter((slot) => !orphans.has(slot)); + } + for (const slot of slots) { + for (const directory of await memberMetadataDirectories(slot)) { + const graphPath = (await loadMetaStrict(directory))?.graphPath; + if (graphPath) referenced.add(path.dirname(path.resolve(directory, graphPath))); } } - slots = slots.filter((slot) => !orphans.has(slot)); - } - for (const slot of slots) { - const graphPath = (await loadMetaStrict(slot))?.graphPath; - if (graphPath) referenced.add(path.dirname(path.resolve(graphPath))); - } - for (const name of await listDirStrict(commitsDir)) { - const dir = path.join(commitsDir, name); - if (referenced.has(dir)) continue; - if (opts.dryRun) { - if (!name.startsWith('.')) result.removed.push(dir); - continue; + for (const name of await listDirStrict(commitsDir)) { + const dir = path.join(commitsDir, name); + if (referenced.has(dir)) continue; + if (opts.dryRun) { + if (!name.startsWith('.')) result.removed.push(dir); + continue; + } + try { + await fs.rm(dir, { recursive: true, force: true }); + if (!name.startsWith('.')) result.removed.push(dir); + } catch { + // Windows refuses to delete a file another process has open (an MCP + // reader). Keep it for the next reclaim instead of failing the caller. + if (!name.startsWith('.')) result.kept.push(dir); + } } - try { - await fs.rm(dir, { recursive: true, force: true }); - if (!name.startsWith('.')) result.removed.push(dir); - } catch { - // Windows refuses to delete a file another process has open (an MCP - // reader). Keep it for the next reclaim instead of failing the caller. - if (!name.startsWith('.')) result.kept.push(dir); - } - } - const isEmpty = async (): Promise => - (await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0; - if (!opts.dryRun && (await isEmpty())) { - // Also hold the cache lock (publish -> cache, the only nesting order) so - // a member saving caches cannot lose them, then re-check: a new member's - // slot may have appeared while waiting. The lock directories live inside - // the store; removing them while held is safe on POSIX and is retried on - // the next reclaim elsewhere. - await withStoreLock({ root: storeRoot }, 'cache', async () => { - if (!(await isEmpty())) return; - await fs - .rm(storeRoot, { recursive: true, force: true }) - .then(() => { - result.storeRemoved = true; - }) - .catch(() => {}); - }); + const isEmpty = async (): Promise => + (await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0; + if (!opts.dryRun && (await isEmpty())) { + // Also hold the cache lock (publish -> cache, the only nesting order) so + // a member saving caches cannot lose them, then re-check: a new member's + // slot may have appeared while waiting. The lock directories live inside + // the store; removing them while held is safe on POSIX and is retried on + // the next reclaim elsewhere. + await withStoreLock({ root: storeRoot }, 'cache', async () => { + if (!(await isEmpty())) return; + await fs + .rm(storeRoot, { recursive: true, force: true }) + .then(() => { + result.storeRemoved = true; + }) + .catch(() => {}); + }); + } + return result; + } finally { + for (const gate of heldMemberGates) gate.release(); } - return result; }; /** @@ -292,9 +373,7 @@ export const describeSharedGraph = ( graphPath: string, storagePath: string, ): 'shared' | 'private' => - path.resolve(graphPath) === path.join(path.resolve(storagePath), LBUG_DIRECTORY) - ? 'private' - : 'shared'; + path.dirname(path.resolve(graphPath)) === path.resolve(storagePath) ? 'private' : 'shared'; /** Files a shared checkout keeps in `/.gitnexus`; everything else there is legacy. */ const POINTER_DIR_KEEP = new Set([SHARED_STORE_POINTER, '.gitignore', 'run.cjs']); diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index a8da36260..50bc3bd4a 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -25,6 +25,7 @@ import { stripWindowsLongPathPrefix } from '../lib/utils.js'; import { getGlobalDir } from './global-dir.js'; import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from './storage-constants.js'; import { slotNameForCanonicalPath, STORAGE_PATH_ENV, STORAGE_ROOT_ENV } from './storage-slot.js'; +import type { RepoMeta } from './repo-meta.js'; export const SHARED_STORE_ENV = 'GITNEXUS_SHARED_STORE'; export const STORES_DIR = 'stores'; @@ -224,10 +225,10 @@ export const storeRootOfCheckoutSlot = (storagePath: string): string | null => { }; /** - * The graph a flat storage slot reads. + * The graph a storage slot reads. * - * Non-shared storage is always `/lbug`, with no I/O. A shared - * checkout slot may record `graphPath` in its metadata, naming a commit graph + * A private generation is an owned basename in the slot. A shared checkout + * slot may record `graphPath` in its metadata, naming a commit graph * in the same store; any other recorded value (outside the store, a sibling's * private slot, unreadable metadata) falls back to the slot's own graph so a * hand-edited file cannot redirect reads. @@ -244,13 +245,31 @@ const readRecordedGraphPath = (slot: string): unknown => { const metaPath = path.resolve(root, INDEX_METADATA_FILE); const rel = path.relative(root, metaPath); if (rel.startsWith('..') || path.isAbsolute(rel)) return undefined; + const cached = recordedGraphCache.get(metaPath); + if (cached) { + const current = fs.lstatSync(metaPath); + if (!current.isFile()) throw new Error('Index metadata must be a regular file'); + if ( + cached.key === + `${current.dev}:${current.ino}:${current.size}:${current.mtimeMs}:${current.ctimeMs}` + ) + return cached.recorded; + } // One open, then stat and read through the same descriptor, so the cache // key always describes the bytes that were parsed. - const fd = fs.openSync(metaPath, 'r'); + const fd = fs.openSync( + metaPath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0), + ); try { const stat = fs.fstatSync(fd); - const key = `${stat.ino}:${stat.size}:${stat.mtimeMs}`; - const cached = recordedGraphCache.get(metaPath); + if (!stat.isFile()) throw new Error('Index metadata must be a regular file'); + if (!fs.constants.O_NOFOLLOW) { + const entry = fs.lstatSync(metaPath); + if (!entry.isFile() || entry.ino !== stat.ino || entry.dev !== stat.dev) + throw new Error('Index metadata changed during open'); + } + const key = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; if (cached?.key === key) return cached.recorded; const recorded = (JSON.parse(fs.readFileSync(fd, 'utf-8')) as { graphPath?: unknown }) .graphPath; @@ -261,17 +280,37 @@ const readRecordedGraphPath = (slot: string): unknown => { } }; -export const resolveGraphPath = (storagePath: string): string => { +export const resolveGraphPath = ( + storagePath: string, + metadata?: Pick, +): string => { const own = path.join(storagePath, LBUG_DIRECTORY); - const root = storeRootOfCheckoutSlot(storagePath); - if (!root) return own; let recorded: unknown; try { - recorded = readRecordedGraphPath(storagePath); + // Capability readers already loaded this receipt; validate that same + // snapshot without another stat, read, or parse on the status hot path. + recorded = metadata === undefined ? readRecordedGraphPath(storagePath) : metadata.graphPath; } catch { return own; } - if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own; + if (typeof recorded !== 'string') return own; + if (/^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) { + const graph = path.join(storagePath, recorded); + const relative = path.relative(storagePath, graph); + if (relative.startsWith('..') || path.isAbsolute(relative)) return own; + // Never follow an edited pointer to a symlink or another slot. A missing + // committed generation must fail at open, rather than serve an older lbug. + try { + const stat = fs.lstatSync(graph); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Published graph generation must be an owned regular file'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + return graph; + } + const root = storeRootOfCheckoutSlot(storagePath); + if (!root || !path.isAbsolute(recorded)) return own; const graph = path.resolve(recorded); const commitDir = path.dirname(graph); // Only a published `-` dir, never `.publish-*` staging. @@ -282,6 +321,15 @@ export const resolveGraphPath = (storagePath: string): string => { return valid ? graph : own; }; +/** A private generation remains its own write target. Only an immutable + * shared commit graph needs a new slot-local file before mutation. */ +export const resolveGraphWritePath = (storagePath: string): string => { + const graph = resolveGraphPath(storagePath); + return path.dirname(path.resolve(graph)) === path.resolve(storagePath) + ? graph + : path.join(storagePath, LBUG_DIRECTORY); +}; + /** * The store slot named by `/.gitnexus/store.json`, or null. The * recorded slot must be this checkout's own slot under the stores directory, diff --git a/gitnexus/src/storage/shared-vector-artifact.ts b/gitnexus/src/storage/shared-vector-artifact.ts new file mode 100644 index 000000000..1ee2aadc7 --- /dev/null +++ b/gitnexus/src/storage/shared-vector-artifact.ts @@ -0,0 +1,191 @@ +/** Retain a completed, immutable vector file in another owned slot. + * Caller holds the source/destination locks (or the shared publish lock). + * Pending artifacts, manifests, journals and mutable ownership never move. */ +import { createHash } from 'node:crypto'; +import { constants, type BigIntStats } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + embeddingJobDirectory, + readEmbeddingJobRecord, + syncEmbeddingJobDirectory, + writeEmbeddingJobRecord, +} from './embedding-job.js'; +import { + assertOwnedVectorGeneration, + embeddingGenerationDirectory, + type PublishedVectorGeneration, +} from './vector-generation.js'; + +type FileIdentity = { dev: string; ino: string; size: string; mtimeNs: string }; +interface RetentionReceipt { + version: 1; + identity?: string; + signature: string; + source: FileIdentity; + /** Absent until the entire retained file is durable. */ + file?: FileIdentity; +} +const facts = (stat: BigIntStats): FileIdentity => ({ + dev: String(stat.dev), + ino: String(stat.ino), + size: String(stat.size), + mtimeNs: String(stat.mtimeNs), +}); +const sameFile = (a: FileIdentity | undefined, b: FileIdentity): boolean => + !!a && (['dev', 'ino', 'size', 'mtimeNs'] as const).every((key) => a[key] === b[key]); + +async function completedFile(file: string): Promise { + const stat = await fs.lstat(file, { bigint: true }); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Retained vectors must be an owned regular file'); + for (const suffix of ['.wal', '.shadow']) { + try { + const sidecar = await fs.lstat(file + suffix); + if (!sidecar.isFile() || sidecar.isSymbolicLink() || sidecar.size) + throw new Error('Cannot retain vectors with unfinished native writes'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + return facts(stat); +} + +async function receipt(directory: string, signature: string): Promise { + const value = (await readEmbeddingJobRecord(directory, 'retained.json')) as RetentionReceipt; + if (!value || value.version !== 1 || value.signature !== signature || !value.source) + throw new Error('Unverified vector retention receipt; artifacts retained'); + return value; +} + +/** A crash can leave the directory before its first atomic receipt rename. + * Only unpublished receipt temporaries are ours to discard in that state. */ +async function recoverEmptyPreparation(directory: string): Promise { + await embeddingJobDirectory(directory); + const temporary = /^\.retained\.json\.[a-f0-9-]{36}\.tmp$/; + for await (const entry of await fs.opendir(directory)) { + const stat = await fs.lstat(path.join(directory, entry.name)); + if (!temporary.test(entry.name) || !stat.isFile() || stat.isSymbolicLink()) + throw new Error('Unverified vector preparation directory; artifacts retained'); + } + // Validate the whole directory before removing anything; stream both passes + // so repeated interrupted attempts cannot require an unbounded entry array. + for await (const entry of await fs.opendir(directory)) { + await fs.unlink(path.join(directory, entry.name)); + } +} + +/** Hard links pin the inode independently of source cleanup. Cross-device + * copies are allowed only for initial local-index adoption, never rebases. + * A directly named preparation directory makes an interrupted copy resumable + * without scanning old jobs or overwriting an unverified existing artifact. */ +export async function retainSharedVectorArtifact( + sourceRoot: string, + destinationRoot: string, + generation: PublishedVectorGeneration, + allowCopy = false, +): Promise { + const artifact = generation.artifactIdentity ?? generation.identity; + const source = await assertOwnedVectorGeneration(sourceRoot, artifact); + const sourceIdentity = await completedFile(source); + const root = await embeddingJobDirectory(destinationRoot); + const directory = embeddingGenerationDirectory(root, artifact); + const parent = path.dirname(directory); + try { + await fs.mkdir(parent, { mode: 0o700 }); + await syncEmbeddingJobDirectory(root); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + await embeddingJobDirectory(parent); + const signature = createHash('sha256') + .update( + JSON.stringify([ + artifact, + generation.vectorIdentity, + generation.inputIdentity, + generation.dimensions, + generation.nodes, + generation.chunks, + generation.indexKind, + ]), + ) + .digest('hex'); + + let present = false; + try { + await fs.lstat(directory); + present = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + if (present) { + const file = await assertOwnedVectorGeneration(root, artifact); + const identity = await completedFile(file); + if ( + file === source || + (sourceIdentity.dev !== '0' && + sourceIdentity.ino !== '0' && + sameFile(identity, sourceIdentity)) + ) + return; + const retained = await receipt(directory, signature); + if (!sameFile(retained.file, identity)) + throw new Error('Retained vector file changed; artifacts retained'); + return; + } + + const preparing = path.join(parent, `.retain-${artifact}`); + let intent: RetentionReceipt; + try { + await fs.mkdir(preparing, { mode: 0o700 }); + intent = { version: 1, identity: artifact, signature, source: sourceIdentity }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + await syncEmbeddingJobDirectory(parent); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + try { + intent = await receipt(preparing, signature); + } catch (readError) { + if ((readError as NodeJS.ErrnoException).code !== 'ENOENT') throw readError; + await recoverEmptyPreparation(preparing); + intent = { version: 1, identity: artifact, signature, source: sourceIdentity }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + await syncEmbeddingJobDirectory(parent); + } + if (!sameFile(intent.source, sourceIdentity)) + throw new Error('Vector retention source changed; artifacts retained'); + } + const target = path.join(preparing, 'vectors'); + if (intent.file) { + if (!sameFile(intent.file, await completedFile(target))) + throw new Error('Prepared vector file changed; artifacts retained'); + } else { + // A verified intent owns only this unacknowledged preparation file. + // Its immutable source remains available even if a copy was interrupted. + try { + await completedFile(target); + await fs.unlink(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + try { + await fs.link(source, target); + } catch (error) { + if (!allowCopy || (error as NodeJS.ErrnoException).code !== 'EXDEV') throw error; + await fs.copyFile(source, target, constants.COPYFILE_EXCL); + const handle = await fs.open(target, 'r+'); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + if (!sameFile(sourceIdentity, await completedFile(source))) + throw new Error('Vector source changed during retention; artifacts retained'); + intent = { ...intent, file: await completedFile(target) }; + await writeEmbeddingJobRecord(preparing, 'retained.json', intent); + } + await fs.rename(preparing, directory); + await syncEmbeddingJobDirectory(parent); +} diff --git a/gitnexus/src/storage/stale-branch-slots.ts b/gitnexus/src/storage/stale-branch-slots.ts index aee92c0ed..49d111f4b 100644 --- a/gitnexus/src/storage/stale-branch-slots.ts +++ b/gitnexus/src/storage/stale-branch-slots.ts @@ -9,11 +9,11 @@ import fs from 'fs/promises'; import path from 'path'; -import { BRANCHES_DIR } from './branch-index.js'; +import { BRANCHES_DIR, branchSlug } from './branch-index.js'; import { listLocalHeads } from './git.js'; import { isMissingFilesystemError, loadMeta } from './repo-meta.js'; import { mapPool } from './map-pool.js'; -import { getStoragePaths, removeBranchIndex } from './repo-manager.js'; +import { removeBranchIndex } from './repo-manager.js'; export type StaleBranchReason = | 'ref-missing' @@ -40,8 +40,8 @@ export interface ListStaleBranchSlotsInput { includeSize?: boolean; } -const slotDirForBranch = (repoPath: string, storagePath: string, branch: string): string => - path.dirname(getStoragePaths(repoPath, branch, storagePath).metaPath); +const slotDirForBranch = (storagePath: string, branch: string): string => + path.join(storagePath, BRANCHES_DIR, branchSlug(branch)); /** Same bound as `listRegisteredRepos`: cap concurrent slot I/O. */ const STALE_SLOT_IO_CONCURRENCY = 8; @@ -191,10 +191,7 @@ export const listStaleBranchSlots = async ( const registryByDir = new Map(); for (const row of recorded) { - registryByDir.set( - path.resolve(slotDirForBranch(input.repoPath, input.storagePath, row.branch)), - row.branch, - ); + registryByDir.set(path.resolve(slotDirForBranch(input.storagePath, row.branch)), row.branch); } const branchesProbe = await probeContainedBranchesRoot(input.storagePath); @@ -519,7 +516,7 @@ export const removeBranchSlot = async ( } const dropRegistry = - dir === null || isSameNormalizedPath(dir, slotDirForBranch(repoPath, storagePath, branch)); + dir === null || isSameNormalizedPath(dir, slotDirForBranch(storagePath, branch)); if (dropRegistry) { try { await removeBranchIndex(repoPath, branch); diff --git a/gitnexus/src/storage/vector-generation.ts b/gitnexus/src/storage/vector-generation.ts new file mode 100644 index 000000000..81384eda5 --- /dev/null +++ b/gitnexus/src/storage/vector-generation.ts @@ -0,0 +1,146 @@ +/** Filesystem-only vector routing. No DB or embedding runtime is loaded here. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { RepoMeta } from './repo-meta.js'; + +export type PublishedVectorGeneration = NonNullable; +export type VectorReadSelection = + | { kind: 'legacy' } + | { kind: 'unavailable'; reason: string } + | { kind: 'ready'; path: string; generation: PublishedVectorGeneration; coverage?: 'preserved' }; +const DIGEST = /^[a-f0-9]{64}$/; +const GENERATION = /^[a-f0-9]{32}$/; +const nonnegative = (n: unknown): n is number => + typeof n === 'number' && Number.isSafeInteger(n) && n >= 0; + +export function embeddingGenerationDirectory(metadataDirectory: string, identity: string): string { + if (!DIGEST.test(identity)) throw new Error('Invalid embedding generation identity'); + return path.join(metadataDirectory, 'embeddings', identity); +} + +/** Metadata must describe the exact graph handle used for result hydration. */ +export function metadataMatchesGraph( + metadataDirectory: string, + metadata: RepoMeta, + graphPath: string, +): boolean { + const recorded = metadata.graphPath; + if (recorded !== undefined && typeof recorded !== 'string') return false; + return path.resolve(metadataDirectory, recorded ?? 'lbug') === path.resolve(graphPath); +} + +/** Advance only a previously bound completed reference. Called under the slot + * lock before invalidating an in-place graph, or while preparing a staged one. + * It neither opens vectors nor upgrades missing/foreign coverage provenance. */ +export function preserveVectorCoverage( + metadata: RepoMeta, + nextGeneration: string, +): RepoMeta['vectorGeneration'] { + if (!GENERATION.test(nextGeneration)) throw new Error('Invalid preservation target'); + const vector = metadata.vectorGeneration; + const graph = metadata.graphGeneration; + if ( + !vector || + vector.version !== 1 || + !GENERATION.test(vector.graphGeneration) || + !graph || + graph.version !== 1 || + !GENERATION.test(graph.id) || + (vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id) + ) + return vector; + return { ...vector, preservedForGraph: nextGeneration }; +} + +/** Run before model initialization, vector COUNTs, or opening an artifact. */ +export function selectVectorGeneration( + metadataDirectory: string, + metadata: RepoMeta | null | undefined, + graphPath: string, + exactScanLimit: number, +): VectorReadSelection { + if (metadata?.incrementalInProgress || metadata?.graphWriteCollapsed) + return { kind: 'unavailable', reason: 'graph-generation-unverified' }; + if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob) + return { kind: 'legacy' }; + const graph = metadata?.graphGeneration; + const vector = metadata?.vectorGeneration; + if ( + !graph || + graph.version !== 1 || + !GENERATION.test(graph.id) || + !DIGEST.test(graph.coveredContent) || + !metadataMatchesGraph(metadataDirectory, metadata, graphPath) + ) + return { kind: 'unavailable', reason: 'graph-generation-unverified' }; + if ( + !vector && + !metadata.embeddingJob && + typeof metadata.legacyVectorSource === 'string' && + DIGEST.test(metadata.legacyVectorSource) && + nonnegative(metadata.stats?.embeddings) && + metadata.stats!.embeddings! > 0 && + nonnegative(metadata.embeddingDims) && + metadata.embeddingDims > 0 + ) + return { kind: 'legacy' }; + if (!vector) + return { + kind: 'unavailable', + reason: metadata.embeddingJob ? 'embedding-pending' : 'embeddings-unavailable', + }; + if (metadata.embeddingJob) return { kind: 'unavailable', reason: 'embedding-pending' }; + if ( + vector.version !== 1 || + !GENERATION.test(vector.graphGeneration) || + (vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id) || + (vector.preservedForGraph !== undefined && !GENERATION.test(vector.preservedForGraph)) || + !DIGEST.test(vector.identity) || + (vector.artifactIdentity !== undefined && !DIGEST.test(vector.artifactIdentity)) || + !DIGEST.test(vector.vectorIdentity) || + !DIGEST.test(vector.inputIdentity) || + !nonnegative(vector.dimensions) || + !vector.dimensions || + !nonnegative(vector.nodes) || + (vector.excludedNodes !== undefined && + (!nonnegative(vector.excludedNodes) || + vector.excludedNodes > vector.nodes || + vector.exclusionReason !== 'input-rejected')) || + !nonnegative(vector.chunks) || + vector.chunks < vector.nodes - (vector.excludedNodes ?? 0) || + (vector.indexKind !== 'hnsw' && vector.indexKind !== 'exact') + ) + return { kind: 'unavailable', reason: 'vector-generation-mismatch' }; + if (!vector.chunks) return { kind: 'unavailable', reason: 'embeddings-empty' }; + if (vector.indexKind === 'exact' && vector.chunks > exactScanLimit) + return { kind: 'unavailable', reason: 'vector-index-pending' }; + return { + kind: 'ready', + path: path.join( + embeddingGenerationDirectory(metadataDirectory, vector.artifactIdentity ?? vector.identity), + 'vectors', + ), + generation: vector, + ...(vector.graphGeneration === graph.id ? {} : { coverage: 'preserved' as const }), + }; +} + +/** Refuse symlinks from the owned metadata slot down to the database. Parent + * path aliases remain valid. The coordinator holds the slot lock while creating + * jobs; published directories are immutable. */ +export async function assertOwnedVectorGeneration( + metadataDirectory: string, + identity: string, +): Promise { + const directory = embeddingGenerationDirectory(metadataDirectory, identity); + for (const target of [metadataDirectory, path.dirname(directory), directory]) { + const stat = await fs.lstat(target); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error('Vector generation must be an owned directory'); + } + const file = path.join(directory, 'vectors'); + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error('Vector generation must be an owned regular file'); + return file; +} diff --git a/gitnexus/test/helpers/analysis-http-server.mjs b/gitnexus/test/helpers/analysis-http-server.mjs new file mode 100644 index 000000000..4f833e8af --- /dev/null +++ b/gitnexus/test/helpers/analysis-http-server.mjs @@ -0,0 +1,11 @@ +import http from 'node:http'; +const listen = http.Server.prototype.listen; +http.Server.prototype.listen = function (...args) { + this.once('listening', () => process.send?.({ port: this.address().port })); + return listen.apply(this, args); +}; +const { createServer } = await import('../../dist/server/api.js'); +await createServer(0); +process.on('message', (message) => { + if (message === 'shutdown') process.emit('SIGTERM'); +}); diff --git a/gitnexus/test/helpers/analysis-process-exit.ts b/gitnexus/test/helpers/analysis-process-exit.ts new file mode 100644 index 000000000..0ba660b42 --- /dev/null +++ b/gitnexus/test/helpers/analysis-process-exit.ts @@ -0,0 +1,31 @@ +import type { ChildProcess } from 'node:child_process'; + +/** Observe real child exit in publication fixtures; IPC is not release evidence. */ +export function waitForAnalysisProcessExit(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + child.removeListener('exit', onExit); + child.removeListener('error', onError); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + if (code === 0 && signal === null) resolve(); + else + reject( + new Error( + `Graph process exited unsuccessfully (${signal ?? code ?? 'unknown'}); prepared artifacts retained`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + if (child.exitCode !== null || child.signalCode !== null) { + onExit(child.exitCode, child.signalCode); + return; + } + child.once('exit', onExit); + child.once('error', onError); + }); +} diff --git a/gitnexus/test/helpers/embedding-restart-runner.mjs b/gitnexus/test/helpers/embedding-restart-runner.mjs new file mode 100644 index 000000000..20f0181f3 --- /dev/null +++ b/gitnexus/test/helpers/embedding-restart-runner.mjs @@ -0,0 +1,110 @@ +// Exit at JS-visible boundaries after native promises settle. Never kill a +// worker or interrupt a native call; omit cleanup to exercise process recovery. +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { runPublishedEmbeddingJob } from '../../dist/core/embeddings/embedding-job.js'; +import { VectorStore } from '../../dist/core/lbug/vector-store.js'; +import { EmbeddingJobJournal } from '../../dist/storage/embedding-job-journal.js'; +import { loadMeta, saveMeta } from '../../dist/storage/repo-manager.js'; + +const [root, boundary] = process.argv.slice(2); +const crash = () => process.exit(73); +const secondUnit = (rows) => rows.some((row) => row.nodeId === 'Function:a.ts:n00064'); +const begin = EmbeddingJobJournal.prototype.begin; +EmbeddingJobJournal.prototype.begin = async function (intent) { + if (intent.start.unit === 1 && boundary === 'intent-before') crash(); + await begin.call(this, intent); + if (intent.start.unit === 1 && boundary === 'intent-after') crash(); +}; +for (const method of ['appendBatch', 'checkpointUnit']) { + const original = VectorStore.prototype[method]; + VectorStore.prototype[method] = async function (rows) { + const selected = secondUnit(rows); + if (selected && boundary === `${method}-before`) crash(); + const result = await original.call(this, rows); + if (selected && boundary === `${method}-after`) crash(); + return result; + }; +} +const acknowledge = EmbeddingJobJournal.prototype.acknowledge; +EmbeddingJobJournal.prototype.acknowledge = async function (result) { + const selected = this.snapshot().active?.start.unit === 1; + if (selected && boundary === 'cursor-before') crash(); + await acknowledge.call(this, result); + if (selected && boundary === 'cursor-after') crash(); +}; +const seal = VectorStore.prototype.seal; +VectorStore.prototype.seal = async function (...args) { + const result = await seal.apply(this, args); + if (boundary === 'seal-after') crash(); + return result; +}; + +const nodes = Array.from({ length: 130 }, (_, index) => { + const name = `n${String(index).padStart(5, '0')}`; + const body = + index === 64 + ? Array.from({ length: 40 }, (_, line) => ` const value${line} = "漢😀";`).join('\n') + : ' return 1;'; + const content = `export function ${name}() {\n${body}\n}`; + return { + id: `Function:a.ts:${name}`, + name, + label: 'Function', + filePath: 'a.ts', + content, + startLine: 1, + endLine: content.split('\n').length, + }; +}); +const byId = new Map(nodes.map((node) => [node.id, node])); +const result = await runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, `lbug.generation.${'a'.repeat(32)}`), + resources: { + graphPoolBytes: 16 * 1024 ** 2, + vectorPoolBytes: 64 * 1024 ** 2, + modelBytes: 0, + modelHeapMb: 0, + queueBytes: 16 * 1024 ** 2, + nativeThreads: 1, + reservedBytes: 96 * 1024 ** 2, + }, + config: { threads: 1, subBatchSize: 8, maxInputChars: 256 }, + metadata: { + loadMetadata: () => loadMeta(root), + saveMetadata: async (metadata) => { + await saveMeta(root, metadata); + if (metadata.vectorGeneration && boundary === 'pointer-after') crash(); + }, + }, + }, + { + identity: () => ({ provider: 'local', model: 'restart-fixture', dimensions: 4 }), + initializeProvider: async () => {}, + openSource: async () => ({ + async *scan(label) { + if (label !== 'Function') return; + for (const node of nodes) + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }, + fetch: async (_label, ids) => ids.map((id) => byId.get(id)), + close: async () => {}, + }), + provider: async (texts) => { + const inputs = texts.map((text) => { + // Interior slices of the oversized node need not repeat its name. + const node = text.match(/n\d{5}/)?.[0] ?? null; + return { node, hash: createHash('sha256').update(text).digest('hex') }; + }); + await fs.appendFile( + path.join(root, `${boundary}.inputs.jsonl`), + inputs.map((input) => JSON.stringify(input) + '\n').join(''), + ); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }, + }, +); +await fs.writeFile(path.join(root, `${boundary}.result.json`), JSON.stringify(result)); diff --git a/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs b/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs new file mode 100644 index 000000000..05efe9e9c --- /dev/null +++ b/gitnexus/test/helpers/interrupted-vector-bootstrap.mjs @@ -0,0 +1,13 @@ +// Exit after native DDL returns to JS, before the artifact identity is written. +// The parent reopens the real file; no storage or recovery implementation is mocked. +import lbug from '@ladybugdb/core'; +import { VectorStore } from '../../dist/core/lbug/vector-store.js'; + +const query = lbug.Connection.prototype.query; +lbug.Connection.prototype.query = async function (sql, ...args) { + const result = await query.call(this, sql, ...args); + if (sql.startsWith('CREATE NODE TABLE CodeEmbedding')) process.exit(73); + return result; +}; +await VectorStore.open(JSON.parse(process.argv[2])); +throw new Error('Expected interruption during vector schema creation'); diff --git a/gitnexus/test/helpers/prepared-analysis-runner.mjs b/gitnexus/test/helpers/prepared-analysis-runner.mjs new file mode 100644 index 000000000..cedbb9fa7 --- /dev/null +++ b/gitnexus/test/helpers/prepared-analysis-runner.mjs @@ -0,0 +1,48 @@ +// Real exit-only writer: the test controls exit only after all graph work settles. +import { runFullAnalysis } from '../../dist/core/run-analyze.js'; +import { resolveAnalysisResourceBudget } from '../../dist/core/analysis-resource-budget.js'; + +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, +}); +const recovering = process.argv[3] === 'recover'; +const phases = []; +const result = await runFullAnalysis( + process.argv[2], + { + graphPublication: recovering ? 'auto' : 'after-exit', + skipNativeCloseOnExit: !recovering, + resources, + skipFts: true, + skipSkills: true, + skipAgentsMd: true, + }, + { + onProgress(phase) { + phases.push(phase); + }, + }, +); +if (recovering) { + await new Promise((resolve, reject) => + process.send( + { alreadyUpToDate: result.alreadyUpToDate, reasons: result.rebuildReasons, phases }, + (error) => (error ? reject(error) : resolve()), + ), + ); + process.exit(0); +} +if (!result.preparedGraph) throw new Error('Expected a prepared graph'); +process.once('message', (message) => { + if (message === 'exit') process.exit(0); +}); +await new Promise((resolve, reject) => + process.send({ handoff: result.preparedGraph, stats: result.stats }, (error) => + error ? reject(error) : resolve(), + ), +); diff --git a/gitnexus/test/integration/analysis-embedding-flow.test.ts b/gitnexus/test/integration/analysis-embedding-flow.test.ts new file mode 100644 index 000000000..0859e5abe --- /dev/null +++ b/gitnexus/test/integration/analysis-embedding-flow.test.ts @@ -0,0 +1,192 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer, type Server } from 'node:http'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { loadMeta } from '../../src/storage/repo-manager.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; + +let root: string | undefined; +let server: Server | undefined; +afterEach(async () => { + if (server) await new Promise((resolve) => server!.close(() => resolve())); + server = undefined; + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + if (root) await fs.rm(root, { recursive: true, force: true }); + root = undefined; +}); + +it('publishes before HTTP work and resumes without parsing or repeating durable requests', async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-flow-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + const content = Array.from( + { length: 130 }, + (_, index) => `export function alpha${String(index).padStart(3, '0')}() { return ${index}; }`, + ).join('\n'); + await fs.writeFile(path.join(repo, 'a.ts'), content); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + // Retrying pending embeddings must also reuse an unchanged dirty graph. + await fs.writeFile(path.join(repo, 'a.ts'), `${content}\n// dirty snapshot\n`); + const accepted: string[] = []; + let fail = true; + let requests = 0; + let graphWasReady = false; + let graphWasVisible = false; + let dimensions = 4; + const endpointErrors: unknown[] = []; + server = createServer(async (request, response) => { + try { + requests++; + expect(graphWasReady).toBe(true); + expect(graphWasVisible).toBe(true); + const published = await loadMeta(slot); + expect(published?.graphGeneration).toBeDefined(); + expect(published?.incrementalInProgress).toBeUndefined(); + expect(published?.embeddingJob).toBeDefined(); + let body = ''; + for await (const chunk of request) body += chunk; + const { input } = JSON.parse(body) as { input: string[] }; + if (fail && accepted.length >= 64) { + response.writeHead(503).end('temporarily unavailable'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: input.map((_text, index) => ({ + index, + embedding: Array.from({ length: dimensions }, (_, dimension) => + dimension === 0 ? 1 : 0, + ), + })), + }), + ); + } catch (error) { + endpointErrors.push(error); + response.writeHead(500).end('fixture failure'); + } + }); + await new Promise((resolve) => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing HTTP fixture port'); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', `http://127.0.0.1:${address.port}/v1`); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'durable-fixture'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '1'); + vi.stubEnv('GITNEXUS_EMBEDDING_SUB_BATCH_SIZE', '8'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, + }); + const options = { + graphPublication: 'inline' as const, + resources, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, + embeddings: true, + embeddingsNodeLimit: 0, + }; + const phases: string[] = []; + let graphPublished!: () => void; + const published = new Promise((resolve) => { + graphPublished = resolve; + }); + let releasePublication!: () => void; + const publicationHeld = new Promise((resolve) => { + releasePublication = resolve; + }); + const callbacks = { + async onGraphPublished() { + // A serving frontend can finish its asynchronous reader refresh first. + graphPublished(); + await publicationHeld; + graphWasVisible = true; + }, + onProgress(phase: string) { + phases.push(phase); + if (phase === 'graph-ready') graphWasReady = true; + }, + }; + const firstRun = runFullAnalysis(repo, options, callbacks); + void firstRun.catch(() => {}); + await Promise.race([published, firstRun]); + expect(requests).toBe(0); + releasePublication(); + await expect(firstRun).rejects.toThrow(/unfinished|incomplete/); + expect(endpointErrors).toEqual([]); + expect(accepted).toHaveLength(64); + const interrupted = await loadMeta(slot); + expect(interrupted?.graphGeneration).toBeDefined(); + expect(interrupted?.embeddingJob).toBeDefined(); + const source = await EmbeddingGraphSource.open( + path.join(slot, interrupted!.graphPath!), + resources.embedding.graphPoolBytes, + 1, + ); + try { + const functions = []; + for await (const key of source.scan('Function')) functions.push(key.id); + expect(functions).toHaveLength(130); + } finally { + await source.close(); + } + + fail = false; + // A long-lived caller honors the existing provider cooldown before retrying. + const now = Date.now; + vi.spyOn(Date, 'now').mockImplementation(() => now() + 31_000); + phases.length = 0; + // Plain analyze resumes the pending job, without requiring --embeddings. + await runFullAnalysis(repo, { ...options, embeddings: false }, callbacks); + expect(endpointErrors).toEqual([]); + expect(phases).not.toContain('extracting'); + expect(new Set(accepted).size).toBe(accepted.length); + expect(accepted).toHaveLength(130); + const completed = await loadMeta(slot); + expect(completed?.graphGeneration).toEqual(interrupted?.graphGeneration); + expect(completed?.embeddingJob).toBeUndefined(); + expect(completed?.vectorGeneration?.chunks).toBe(130); + const count = requests; + phases.length = 0; + const cached = await runFullAnalysis(repo, options, callbacks); + expect(cached.alreadyUpToDate).toBe(true); + expect(phases).not.toContain('extracting'); + expect(await loadMeta(slot)).toEqual(completed); + expect(requests).toBe(count); + + // A vector-space change regenerates vectors without parsing or republishing + // the graph. Its legacy CodeEmbedding column is no longer the vector store. + dimensions = 8; + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', String(dimensions)); + phases.length = 0; + await runFullAnalysis(repo, options, callbacks); + expect(phases).not.toContain('extracting'); + const resized = await loadMeta(slot); + expect(resized?.graphGeneration).toEqual(completed?.graphGeneration); + expect(resized?.vectorGeneration?.dimensions).toBe(dimensions); + expect(resized?.vectorGeneration?.identity).not.toBe(completed?.vectorGeneration?.identity); + expect(accepted).toHaveLength(260); + + await fs.writeFile(path.join(repo, 'a.ts'), content.replace('return 0;', 'return 999;')); + await runFullAnalysis(repo, options, callbacks); + expect(accepted).toHaveLength(261); + expect(accepted.at(-1)).toContain('return 999;'); + expect((await loadMeta(slot))?.graphGeneration?.id).not.toBe(completed?.graphGeneration?.id); +}, 120_000); diff --git a/gitnexus/test/integration/analysis-generation-routing.test.ts b/gitnexus/test/integration/analysis-generation-routing.test.ts new file mode 100644 index 000000000..c302a7742 --- /dev/null +++ b/gitnexus/test/integration/analysis-generation-routing.test.ts @@ -0,0 +1,182 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { ensurePrivateSharedGraph } from '../../src/core/shared-store-analyze.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import * as nativeResources from '../../src/core/lbug/lbug-adapter.js'; +import * as modelResources from '../../src/core/embeddings/embedding-sidecar-client.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { resolveSharedStore } from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +let root: string; +let repo: string; +const generation = `lbug.generation.${'a'.repeat(32)}`; +const options = { skipFts: true, skipAgentsMd: true, skipSkills: true }; +const git = (...args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8', stdio: 'pipe' }).trim(); +const metadata = (extra: Partial = {}): RepoMeta => ({ + repoPath: repo, + lastCommit: git('rev-parse', 'HEAD'), + indexedAt: '2026-11-04T00:00:00Z', + branch: 'main', + ...extra, +}); + +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-generation-routing-'))); + repo = path.join(root, 'repo'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + git('init', '-q', '-b', 'main'); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 1; }\n'); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); +}); +afterEach(async () => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +it.each([false, true])( + 'retains the owned graph filename on a tiny edit (shared=%s)', + async (shared) => { + if (shared) { + git('worktree', 'add', '-q', '-b', 'sibling', path.join(root, 'sibling')); + delete process.env.GITNEXUS_SHARED_STORE; + } + const first = await runFullAnalysis(repo, options, { onProgress: () => {} }); + const slot = first.storagePath; + if (shared) expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + const prior = await loadMeta(slot); + if (!prior) throw new Error('Fixture index is missing'); + const owned = path.join(slot, generation); + await fs.rename(path.join(slot, 'lbug'), owned); + await saveMeta(slot, { ...prior, graphPath: generation }); + const inode = (await fs.stat(owned)).ino; + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 2; }\n'); + const copy = vi.spyOn(fs, 'copyFile'); + const result = await runFullAnalysis(repo, options, { onProgress: () => {} }); + expect(result.alreadyUpToDate).not.toBe(true); + expect((await loadMeta(slot))?.graphPath).toBe(generation); + expect(getStoragePaths(repo, undefined, slot).lbugPath).toBe(owned); + expect((await fs.stat(owned)).ino).toBe(inode); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect( + copy.mock.calls.filter((args) => + args.some((file) => path.basename(String(file)).startsWith('lbug')), + ), + ).toHaveLength(0); + const source = await EmbeddingGraphSource.open(owned, 256 * 1024 * 1024, 1); + try { + const ids: string[] = []; + for await (const key of source.scan('Function')) ids.push(key.id); + expect(ids).toHaveLength(1); + expect((await source.fetch('Function', ids))[0].content).toContain('return 2'); + } finally { + await source.close(); + } + }, + 120_000, +); + +it.each(['pending', 'prepared', 'branch-pending'] as const)( + 'takes the original slot lock before shared adoption with %s work', + async (kind) => { + const local = path.join(repo, '.gitnexus'); + await saveMeta(local, metadata()); + let lockedSlot = local; + if (kind === 'prepared') { + // Even unreadable publication evidence must not be stranded by routing. + await fs.writeFile(path.join(local, 'analysis-publication.json'), '{'); + } else { + if (kind === 'branch-pending') { + git('checkout', '-q', '-b', 'feature'); + lockedSlot = path.dirname(getStoragePaths(repo, 'feature', local).metaPath); + } + await saveMeta( + lockedSlot, + metadata({ + branch: kind === 'branch-pending' ? 'feature' : 'main', + embeddingJob: { version: 1, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }, + }), + ); + } + git('worktree', 'add', '-q', '-b', 'sibling', path.join(root, 'sibling')); + delete process.env.GITNEXUS_SHARED_STORE; + const shared = resolveSharedStore(repo); + if (!shared) throw new Error('Fixture store is missing'); + const lock = await acquireIndexLock(lockedSlot, { sweep: false }); + const waitedForOwner = new Error('Observed the original owner lock'); + try { + await expect( + runFullAnalysis( + repo, + { + ...options, + ...(kind === 'branch-pending' ? { branch: 'feature' } : {}), + }, + { + onProgress: (phase) => { + if (phase === 'lock') throw waitedForOwner; + throw new Error(`Analysis bypassed its retained work: ${phase}`); + }, + }, + ), + ).rejects.toBe(waitedForOwner); + expect(await loadMeta(shared.checkoutSlot)).toBeNull(); + if (kind !== 'prepared') expect((await loadMeta(lockedSlot))?.embeddingJob).toBeDefined(); + } finally { + lock.release(); + } + }, + 30_000, +); + +it('leaves model resources alone on a no-op, then drains them before actual graph work', async () => { + await runFullAnalysis(repo, options, { onProgress: () => {} }); + const events: string[] = []; + const native = vi + .spyOn(nativeResources, 'drainAnalysisNativeResources') + .mockImplementation(async () => { + events.push('native'); + }); + const model = vi + .spyOn(modelResources, 'drainEmbeddingSidecarForAnalysis') + .mockImplementation(async () => { + events.push('model'); + }); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, + }); + const noop = await runFullAnalysis(repo, { ...options, resources }, { onProgress: () => {} }); + expect(noop.alreadyUpToDate).toBe(true); + expect(native).not.toHaveBeenCalled(); + expect(model).not.toHaveBeenCalled(); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 3; }\n'); + await runFullAnalysis( + repo, + { ...options, resources }, + { + onProgress: (phase) => { + if (phase === 'extracting') { + expect(events.slice(0, 2)).toEqual(['model', 'native']); + events.push('parse'); + } + }, + }, + ); + expect(events).toContain('parse'); +}, 120_000); diff --git a/gitnexus/test/integration/analysis-publication-flow.test.ts b/gitnexus/test/integration/analysis-publication-flow.test.ts new file mode 100644 index 000000000..a6f5cc657 --- /dev/null +++ b/gitnexus/test/integration/analysis-publication-flow.test.ts @@ -0,0 +1,510 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fork, execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { continuePreparedAnalysis, runFullAnalysis } from '../../src/core/run-analyze.js'; +import { verifyPublishedAnalysisGraph } from '../../src/core/analysis-graph-verification.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { + readPreparedAnalysis, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import type { PreparedAnalysisHandoff } from '../../src/core/analysis-phase-supervisor.js'; +import { waitForAnalysisProcessExit } from '../helpers/analysis-process-exit.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import lbug from '@ladybugdb/core'; +import { + openLbugConnection, + closeLbugConnection, + estimateBufferPool, +} from '../../src/core/lbug/lbug-config.js'; +import { readLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; + +let root: string, repo: string, slot: string; +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4 * 1024 ** 3, + availableCpus: 2, + totalMb: 2048, + workers: 1, + graphPoolBytes: 256 * 1024 ** 2, + modelMemoryMb: 0, +}); +const options = { + graphPublication: 'inline' as const, + resources, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, +}; +const callbacks = { onProgress() {} }; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publication-flow-'))); + repo = path.join(root, 'repo'); + slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 1; }\n'); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); +}); +async function meta(): Promise { + const result = await loadMeta(slot); + if (!result) throw new Error('Missing fixture metadata'); + return result; +} +async function contents(metadata: RepoMeta) { + const source = await EmbeddingGraphSource.open( + path.join(slot, metadata.graphPath!), + 256 * 1024 ** 2, + 1, + ); + try { + const ids = []; + for await (const key of source.scan('Function')) ids.push(key.id); + return (await source.fetch('Function', ids))[0].content; + } finally { + await source.close(); + } +} + +it('keeps automatic plain graph analysis on its existing publication path', async () => { + const beforeWork = vi.fn(() => { + throw new Error('Unexpected process handoff'); + }); + await runFullAnalysis( + repo, + { ...options, graphPublication: 'auto' }, + { + ...callbacks, + onGraphWork: beforeWork, + }, + ); + expect(beforeWork).not.toHaveBeenCalled(); + expect((await meta()).graphGeneration).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'lbug'))).resolves.toBeDefined(); +}, 120_000); + +it.each([128, 1024])( + 'keeps a small graph cache within a %i MiB ceiling', + async (poolMb) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', undefined); + const Database = lbug.Database; + const opened: Array<{ file: string; pool: number }> = []; + vi.spyOn(lbug, 'Database').mockImplementation(function (...args) { + opened.push({ file: args[0], pool: args[1]! }); + return Reflect.construct(Database, args); + }); + const roomy = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 2, + totalMb: 4096, + workers: 1, + graphPoolBytes: poolMb * 1024 ** 2, + modelMemoryMb: 0, + }); + await runFullAnalysis( + repo, + { ...options, graphPublication: 'auto', resources: roomy }, + callbacks, + ); + const writers = opened.filter(({ file }) => file.startsWith(path.join(slot, 'lbug'))); + expect(writers.length).toBeGreaterThan(0); + const expected = Math.min(estimateBufferPool(100), poolMb * 1024 ** 2); + expect(writers.every(({ pool }) => pool === expected)).toBe(true); + }, + 120_000, +); + +it('requests an exit supervisor before parsing or opening a graph writer', async () => { + const phases: string[] = []; + await expect( + runFullAnalysis( + repo, + { + ...options, + graphPublication: 'auto', + embeddings: true, + skipNativeCloseOnExit: true, + }, + { + onProgress: (phase) => phases.push(phase), + onGraphWork: () => { + throw new Error('Supervisor requested'); + }, + }, + ), + ).rejects.toThrow('Supervisor requested'); + expect(phases).not.toContain('extracting'); + expect(await loadMeta(slot)).toMatchObject({ repoPath: repo, lastCommit: '' }); + expect((await loadMeta(slot))?.graphGeneration).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); +}, 120_000); + +it.each([true, false])( + 'publishes a verified native graph before reporting graph-ready (skipFts=%s)', + async (skipFts) => { + const events: string[] = []; + await runFullAnalysis( + repo, + { ...options, skipFts }, + { + onProgress(phase, _percent, _message, details) { + if (phase === 'graph-ready') { + expect(details?.graphPhase).toBe('ready'); + events.push(phase); + } + if (phase === 'done' && _percent === 100) events.push('terminal'); + }, + }, + ); + const published = await meta(); + if (!skipFts) expect(published.capabilities?.fts.status).toBe('available'); + expect(events).toEqual(['graph-ready']); + expect(published.graphGeneration?.coveredContent).toMatch(/^[a-f0-9]{64}$/); + expect(published.graphPath).toMatch(/^lbug\.generation\.[a-f0-9]{32}$/); + expect(await contents(published)).toContain('return 1'); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect( + verifyPublishedAnalysisGraph( + path.join(slot, published.graphPath!), + { + ...published, + stats: { ...published.stats, nodes: (published.stats?.nodes ?? 0) + 1 }, + }, + { bufferPoolBytes: 256 * 1024 ** 2, threads: 1 }, + ), + ).rejects.toThrow('counts differ'); + }, + 120_000, +); + +it('advances the logical generation on a tiny edit without copying its private graph', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + const graph = path.join(slot, first.graphPath!); + const inode = (await fs.stat(graph)).ino; + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 2; }\n'); + const copy = vi.spyOn(fs, 'copyFile'); + await runFullAnalysis(repo, options, callbacks); + const second = await meta(); + expect(second.graphPath).toBe(first.graphPath); + expect((await fs.stat(graph)).ino).toBe(inode); + expect(second.graphGeneration?.id).not.toBe(first.graphGeneration?.id); + expect(second.graphGeneration?.coveredContent).not.toBe(first.graphGeneration?.coveredContent); + expect(second.incrementalInProgress).toBeUndefined(); + expect(await contents(second)).toContain('return 2'); + expect( + copy.mock.calls.filter((args) => + args.some((file) => path.basename(String(file)).startsWith('lbug')), + ), + ).toHaveLength(0); + const unchanged = await runFullAnalysis(repo, options, callbacks); + expect(unchanged.alreadyUpToDate).toBe(true); + expect((await meta()).graphGeneration?.id).toBe(second.graphGeneration?.id); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 5; }\n'); + expect((await runFullAnalysis(repo, options, callbacks)).alreadyUpToDate).not.toBe(true); + expect(await contents(await meta())).toContain('return 5'); + await fs.writeFile(path.join(repo, 'b.ts'), 'export function beta() { return 6; }\n'); + expect((await runFullAnalysis(repo, options, callbacks)).alreadyUpToDate).not.toBe(true); + expect((await meta()).fileHashes?.['b.ts']).toBeDefined(); +}, 120_000); + +it('verifies published FTS with an empty extension home', async () => { + const home = path.join(root, 'empty-home'); + await fs.mkdir(home); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + await runFullAnalysis(repo, { ...options, skipFts: false }, callbacks); + const published = await meta(); + expect(published.capabilities?.fts.status).toBe('available'); + expect(published.graphGeneration).toBeDefined(); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect(fs.stat(path.join(home, '.lbdb', 'extension'))).rejects.toMatchObject({ + code: 'ENOENT', + }); +}, 120_000); + +it('preserves completed vector coverage across a new graph generation', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + const vector = { + version: 1 as const, + graphGeneration: first.graphGeneration!.id, + identity: 'a'.repeat(64), + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact' as const, + }; + await saveMeta(slot, { ...first, vectorGeneration: vector }); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 3; }\n'); + await runFullAnalysis(repo, options, callbacks); + const second = await meta(); + expect(second.vectorGeneration).toEqual({ + ...vector, + preservedForGraph: second.graphGeneration?.id, + }); + expect(second.stats?.embeddings).toBe(1); +}, 120_000); + +it('preserves an old-width legacy table before writable schema initialization changes it', async () => { + await runFullAnalysis(repo, { ...options, graphPublication: 'auto' }, callbacks); + const graph = path.join(slot, 'lbug'); + const handle = await openLbugConnection(lbug, graph, { + bufferPoolBytes: 256 * 1024 ** 2, + threads: 1, + }); + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) cursor.close(); + }; + try { + await query('DROP TABLE CodeEmbedding'); + await query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ); + await query( + "CREATE (:CodeEmbedding {id:'old:0',nodeId:'old',chunkIndex:0,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'historical'})", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: graph }); + } + await saveMeta(slot, { + ...(await meta()), + embeddingDims: 4, + stats: { ...(await meta()).stats, embeddings: 1 }, + }); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 7; }\n'); + await runFullAnalysis(repo, options, callbacks); + const published = await meta(); + expect(published.legacyVectorSource).toMatch(/^[a-f0-9]{64}$/); + const retained = await readLegacyEmbeddingSource(slot, published.legacyVectorSource!); + expect(retained.recordedIdentity).toBeUndefined(); + const old = await openLbugConnection(lbug, retained.path, { + readOnly: true, + bufferPoolBytes: 32 * 1024 ** 2, + threads: 1, + }); + try { + const result = await old.conn.query('MATCH (e:CodeEmbedding) RETURN e.embedding AS vector'); + const cursors = Array.isArray(result) ? result : [result]; + try { + expect(await cursors[0].getAll()).toEqual([{ vector: [1, 0, 0, 0] }]); + } finally { + for (const cursor of cursors) cursor.close(); + } + } finally { + await closeLbugConnection(old, { strict: true, databasePath: retained.path }); + } + expect(await contents(published)).toContain('return 7'); +}, 120_000); + +it('retains an unpublished replacement and its parse cache when publication fails', async () => { + await runFullAnalysis(repo, options, callbacks); + const first = await meta(); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function alpha() { return 4; }\n'); + const rename = fs.rename; + const failPublication = vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + if ( + String(source).includes('.staging.') && + path.basename(String(destination)).startsWith('lbug.generation.') + ) + throw new Error('publication interrupted'); + return rename(source, destination); + }); + await expect(runFullAnalysis(repo, { ...options, force: true }, callbacks)).rejects.toThrow( + 'publication interrupted', + ); + failPublication.mockRestore(); + expect((await meta()).graphGeneration?.id).toBe(first.graphGeneration?.id); + expect(await contents(await meta())).toContain('return 1'); + const pending = await readPreparedAnalysis(slot); + expect(pending).toBeDefined(); + await expect(fs.stat(path.join(slot, pending!.stagingFile))).resolves.toBeDefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${pending!.id}.json`)), + ).resolves.toBeDefined(); + expect((await fs.readdir(path.join(slot, 'parse-rebuild'))).length).toBeGreaterThan(0); + const recoveryLogs: string[] = []; + const result = await runFullAnalysis(repo, options, { + onLog: (message) => { + recoveryLogs.push(message); + }, + onProgress(phase) { + if (phase === 'extracting') + throw new Error(`Recovery unexpectedly reparsed: ${recoveryLogs.join('\n')}`); + }, + }); + expect(result.alreadyUpToDate).toBe(true); + expect((await meta()).graphGeneration?.id).toBe(pending!.id); + expect(await contents(await meta())).toContain('return 4'); +}, 120_000); + +it.each(['prepared', 'graph-published'] as const)( + 'recovers the %s graph and parse stage after actual writer exit without parsing', + async (state) => { + const child = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + execArgv: [], + env: process.env, + }, + ); + const exit = waitForAnalysisProcessExit(child); + void exit.catch(() => {}); + let output = ''; + child.stderr?.on('data', (chunk) => { + output = (output + String(chunk)).slice(-8192); + }); + const prepared = new Promise<{ handoff: { publicationId: string } }>((resolve, reject) => { + child.once('message', (message) => + resolve(message as { handoff: { publicationId: string } }), + ); + child.once('error', reject); + child.once('exit', (code) => { + if (code !== 0) reject(new Error(`Graph child exited ${code}: ${output}`)); + }); + }); + try { + const message = await prepared; + expect((await readPreparedAnalysis(slot))?.id).toBe(message.handoff.publicationId); + expect((await meta()).graphGeneration).toBeUndefined(); + expect((await fs.readdir(path.join(slot, 'parse-rebuild'))).length).toBeGreaterThan(0); + await expect(runFullAnalysis(repo, options, callbacks)).rejects.toThrow('has not exited'); + child.send('exit'); + await exit; + if (state === 'graph-published') { + // Simulate exit between durable graph publication and cache publication. + await publishPreparedAnalysis(slot, { + saveMetadata: (metadata) => saveMeta(slot, metadata), + verify: (file, metadata) => + verifyPublishedAnalysisGraph(file, metadata, { + bufferPoolBytes: resources.parse.graphPoolBytes, + threads: resources.parse.nativeThreads, + }), + }); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${message.handoff.publicationId}.json`)), + ).resolves.toBeDefined(); + } + // Use the same distribution runtime for both phases: mixing the Vitest and + // plain Node parser loaders would test an analyzer change as well as recovery. + const continuation = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo, 'recover'], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + execArgv: [], + env: process.env, + }, + ); + const continuationExit = waitForAnalysisProcessExit(continuation); + const result = await new Promise<{ + alreadyUpToDate: boolean; + reasons: string[]; + phases: string[]; + }>((resolve, reject) => { + continuation.once('message', (message) => + resolve(message as { alreadyUpToDate: boolean; reasons: string[]; phases: string[] }), + ); + continuation.once('error', reject); + void continuationExit.catch(reject); + }); + await continuationExit; + expect({ alreadyUpToDate: result.alreadyUpToDate, reasons: result.reasons }).toEqual({ + alreadyUpToDate: true, + reasons: [], + }); + expect(result.phases).not.toContain('extracting'); + expect((await meta()).graphGeneration?.id).toBe(message.handoff.publicationId); + expect(await contents(await meta())).toContain('return 1'); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + const keys = (await meta()).cacheKeys!; + expect(keys.length).toBeGreaterThan(0); + for (const key of keys) + await expect(fs.stat(path.join(slot, 'parse-cache', `${key}.v8`))).resolves.toBeDefined(); + await expect( + fs.stat(path.join(slot, `analysis-cache-${message.handoff.publicationId}.json`)), + ).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + if (child.connected) child.send('exit'); + await exit; + } + }, + 120_000, +); + +it('continues only the exited writer and unchanged target, without repeating forced parsing', async () => { + const child = fork( + fileURLToPath(new URL('../helpers/prepared-analysis-runner.mjs', import.meta.url)), + [repo], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'], execArgv: [], env: process.env }, + ); + const exit = waitForAnalysisProcessExit(child); + void exit.catch(() => {}); + child.stderr?.resume(); + child.stdout?.resume(); + try { + const { handoff } = await new Promise<{ handoff: PreparedAnalysisHandoff }>( + (resolve, reject) => { + child.once('message', (message) => + resolve(message as { handoff: PreparedAnalysisHandoff }), + ); + child.once('error', reject); + void exit.then(() => reject(new Error('Writer exited without preparing')), reject); + }, + ); + if (!child.pid) throw new Error('Missing writer PID'); + await expect(continuePreparedAnalysis(handoff, child.pid, options, callbacks)).rejects.toThrow( + 'has not exited', + ); + child.send('exit'); + await exit; + await expect( + continuePreparedAnalysis( + handoff, + child.pid, + { ...options, registryName: 'changed' }, + callbacks, + ), + ).rejects.toThrow('registered name changed'); + expect((await readPreparedAnalysis(slot))?.id).toBe(handoff.publicationId); + const phases: string[] = []; + const published = await continuePreparedAnalysis( + handoff, + child.pid, + { ...options, force: true }, + { onProgress: (phase) => phases.push(phase) }, + ); + expect(phases).toEqual(['graph-ready', 'done']); + expect(published.graphGeneration?.id).toBe(handoff.publicationId); + expect(await contents(published)).toContain('return 1'); + expect(await readPreparedAnalysis(slot)).toBeUndefined(); + await expect(fs.stat(path.join(slot, 'parse-rebuild'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + } finally { + if (child.connected) child.send('exit'); + await exit; + } +}, 120_000); diff --git a/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts b/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts new file mode 100644 index 000000000..80bb5c913 --- /dev/null +++ b/gitnexus/test/integration/analyze-cli-embedding-phases.test.ts @@ -0,0 +1,184 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer } from 'node:http'; +import { execFileSync, fork, spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { loadMeta } from '../../src/storage/repo-manager.js'; +import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; + +it.each(['CLI', 'CLI pinned heap', 'auto-sync', 'embeddings sync'] as const)( + '%s publishes after writer exit and resumes only unfinished HTTP embeddings', + async (frontend) => { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-cli-phases-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + const content = Array.from( + { length: 130 }, + (_, i) => `export function alpha${String(i).padStart(3, '0')}() { return ${i}; }`, + ).join('\n'); + await fs.writeFile(path.join(repo, 'a.ts'), content); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + let fail = true; + const accepted: string[] = []; + const observations: unknown[] = []; + const errors: unknown[] = []; + const server = createServer(async (request, response) => { + try { + const meta = await loadMeta(slot); + expect(meta?.graphGeneration).toBeDefined(); + expect(meta?.incrementalInProgress).toBeUndefined(); + expect(meta?.embeddingJob).toBeDefined(); + await expect(fs.stat(path.join(slot, 'analysis-publication.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + observations.push(meta?.graphGeneration?.id); + let body = ''; + for await (const chunk of request) body += chunk; + const input: string[] = JSON.parse(body).input; + if (fail && accepted.length >= 64) { + response.writeHead(503).end('outage'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })), + }), + ); + } catch (error) { + errors.push(error); + response.writeHead(500).end('fixture failure'); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing fixture port'); + const env = { + ...process.env, + // This fixture exercises automatic process-family admission and respawning. + GITNEXUS_MEMORY: '', + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_EMBEDDING_URL: `http://127.0.0.1:${address.port}/v1`, + GITNEXUS_EMBEDDING_MODEL: 'cli-phase-fixture', + GITNEXUS_EMBEDDING_DIMS: '4', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '8', + }; + const runCli = (embeddings: boolean, sync = false) => + new Promise<{ code: number | null; output: string }>((resolve, reject) => { + const child = spawn( + process.execPath, + [ + ...(sync ? ['--max-old-space-size=1024'] : []), + ...(frontend === 'CLI pinned heap' && !sync + ? ['--max-old-space-size=512', '--max-semi-space-size=16'] + : []), + fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url)), + ...(sync + ? ['embeddings', 'sync', repo] + : [ + 'analyze', + repo, + '--index-only', + '--skip-fts', + '--workers', + '1', + '--resource-budget', + '2048', + ...(embeddings ? ['--embeddings', '0'] : []), + ]), + ], + { env, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let output = ''; + const collect = (chunk: Buffer) => { + output = (output + chunk.toString()).slice(-32_768); + }; + child.stdout.on('data', collect); + child.stderr.on('data', collect); + child.once('error', reject); + child.once('close', (code) => resolve({ code, output })); + }); + const runWorker = async (embeddings: boolean) => { + const run = createAutoSyncAnalysisRunner({ + forkWorker: (_workerPath, execArgv) => + fork(fileURLToPath(new URL('../../dist/server/analyze-worker.js', import.meta.url)), [], { + execArgv, + env, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }), + }); + try { + await run( + repo, + { + embeddings, + embeddingsNodeLimit: 0, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, + workerPoolSize: 1, + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 2048 * 1024 ** 2, + totalMb: 2048, + availableCpus: 2, + workers: 1, + modelMemoryMb: 0, + }), + }, + 120_000, + ); + return { code: 0, output: '' }; + } catch (error) { + return { code: 1, output: (error as Error).message }; + } + }; + const run = frontend === 'auto-sync' ? runWorker : runCli; + const resume = frontend === 'embeddings sync' ? () => runCli(false, true) : run; + try { + const first = await run(true); + expect(first.code, first.output).toBe(1); + expect(errors).toEqual([]); + expect(accepted, first.output).toHaveLength(64); + expect(first.output).not.toContain('Repository indexed successfully'); + const graph = await loadMeta(slot); + expect(graph?.embeddingJob).toBeDefined(); + if (!graph?.graphPath) throw new Error('Expected owned graph'); + const inode = (await fs.stat(path.join(slot, graph.graphPath))).ino; + fail = false; + const retry = await resume(false); + expect(retry.code, retry.output).toBe(0); + if (frontend === 'CLI') + expect(retry.output.match(/Repository indexed successfully/g)).toHaveLength(1); + expect(accepted).toHaveLength(130); + expect(new Set(accepted).size).toBe(130); + expect(new Set(observations)).toEqual(new Set([graph.graphGeneration?.id])); + expect((await loadMeta(slot))?.embeddingJob).toBeUndefined(); + const repeat = await resume(true); + expect(repeat.code, repeat.output).toBe(0); + expect(accepted).toHaveLength(130); + await fs.writeFile(path.join(repo, 'a.ts'), content.replace('return 0;', 'return 999;')); + const changed = await run(true); + expect(changed.code, changed.output).toBe(0); + expect(accepted).toHaveLength(131); + const updated = await loadMeta(slot); + expect(updated?.graphGeneration?.id).not.toBe(graph.graphGeneration?.id); + expect(updated?.graphPath).toBe(graph.graphPath); + expect((await fs.stat(path.join(slot, graph.graphPath))).ino).toBe(inode); + expect(errors).toEqual([]); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + } + }, + 180_000, +); diff --git a/gitnexus/test/integration/analyze-http-embedding-phases.test.ts b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts new file mode 100644 index 000000000..2c0c697cb --- /dev/null +++ b/gitnexus/test/integration/analyze-http-embedding-phases.test.ts @@ -0,0 +1,157 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createServer } from 'node:http'; +import { execFileSync, fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { loadMeta } from '../../src/storage/repo-manager.js'; + +it('serves the published graph during an HTTP outage and resumes through /api/embed without repeating paid work', async () => { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-http-phases-'))); + const repo = path.join(root, 'repo'); + const slot = path.join(repo, '.gitnexus'); + await fs.mkdir(repo); + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' }); + git('init', '-q', '-b', 'main'); + await fs.writeFile( + path.join(repo, 'a.ts'), + Array.from( + { length: 130 }, + (_, i) => `export function alpha${String(i).padStart(3, '0')}() { return ${i}; }`, + ).join('\n'), + ); + git('add', 'a.ts'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + let fail = true; + let releaseOutage!: () => void; + const outage = new Promise((resolve) => { + releaseOutage = resolve; + }); + const accepted: string[] = []; + const provider = createServer(async (request, response) => { + let body = ''; + for await (const chunk of request) body += chunk; + const input: string[] = JSON.parse(body).input; + if (fail && accepted.length >= 64) { + await outage; + response.writeHead(503).end('outage'); + return; + } + accepted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ data: input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })) }), + ); + }); + await new Promise((resolve) => provider.listen(0, '127.0.0.1', resolve)); + const providerAddress = provider.address(); + if (!providerAddress || typeof providerAddress === 'string') throw new Error('No provider port'); + const child = fork( + fileURLToPath(new URL('../helpers/analysis-http-server.mjs', import.meta.url)), + [], + { + execArgv: ['--max-old-space-size=1024'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + env: { + ...process.env, + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_SERVER_ANALYZE_HEAP_MB: '512', + GITNEXUS_WORKER_POOL_SIZE: '1', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(256 * 1024 ** 2), + GITNEXUS_EMBEDDING_URL: `http://127.0.0.1:${providerAddress.port}/v1`, + GITNEXUS_EMBEDDING_MODEL: 'http-phase-fixture', + GITNEXUS_EMBEDDING_DIMS: '4', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '8', + }, + }, + ); + let output = ''; + const collect = (chunk: Buffer) => { + output = (output + chunk.toString()).slice(-32768); + }; + child.stdout?.on('data', collect); + child.stderr?.on('data', collect); + const exited = new Promise((resolve) => child.once('close', resolve)); + try { + const port = await new Promise((resolve, reject) => { + child.once('message', (message) => resolve((message as { port: number }).port)); + child.once('error', reject); + void exited.then((code) => reject(new Error(`Server exited ${code}: ${output}`))); + }); + const request = async (route: string, body?: unknown) => { + const response = await fetch(`http://127.0.0.1:${port}${route}`, { + method: body === undefined ? 'GET' : 'POST', + headers: { 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const value = (await response.json()) as Record; + expect(response.ok, JSON.stringify(value) + '\n' + output).toBe(true); + return value; + }; + const until = async (check: () => Promise) => { + const end = Date.now() + 90_000; + while (!(await check())) { + if (Date.now() > end) throw new Error(`Timed out: ${output}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }; + const started = await request('/api/analyze', { path: repo, embeddings: true }); + const poll = () => request(`/api/analyze/${started.jobId}`); + await until(async () => (await poll()).status === 'analyzing'); + const graphQuery = request('/api/query?repo=repo', { + cypher: 'MATCH (n:Function) RETURN count(n) AS count', + }); + await until(async () => accepted.length === 64); + expect((await graphQuery).error).toBeUndefined(); + expect((await poll()).status).toBe('analyzing'); + expect((await loadMeta(slot))?.graphGeneration).toBeDefined(); + const keyword = await request('/api/search?repo=repo', { + query: 'alpha000', + mode: 'bm25', + enrich: false, + }); + expect(JSON.stringify(keyword)).toContain('alpha000'); + releaseOutage(); + await until(async () => (await poll()).status === 'failed'); + expect(accepted).toHaveLength(64); + fail = false; + const retry = await request('/api/embed?repo=repo', {}); + await until(async () => { + const job = await request(`/api/embed/${retry.jobId}`); + if (job.status === 'failed') throw new Error(JSON.stringify(job) + '\n' + output); + return job.status === 'complete'; + }); + expect(accepted).toHaveLength(130); + expect(new Set(accepted).size).toBe(130); + expect((await loadMeta(slot))?.embeddingJob).toBeUndefined(); + const semantic = await request('/api/search?repo=repo', { + query: 'function returns a number', + mode: 'semantic', + enrich: false, + }); + expect(JSON.stringify(semantic)).toContain('alpha'); + const paidInputs = accepted.length; + const generation = (await loadMeta(slot))?.graphGeneration; + const cached = await request('/api/analyze', { path: repo, embeddings: true }); + await until(async () => { + const job = await request(`/api/analyze/${cached.jobId}`); + if (job.status === 'failed') throw new Error(JSON.stringify(job) + '\n' + output); + return job.status === 'complete'; + }); + expect(accepted).toHaveLength(paidInputs); + expect((await loadMeta(slot))?.graphGeneration).toEqual(generation); + } finally { + releaseOutage(); + try { + if (child.connected) child.send('shutdown'); + expect(await exited, output).toBe(0); + } finally { + provider.closeAllConnections(); + await new Promise((resolve) => provider.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + } + } +}, 180_000); diff --git a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts index 33db6b36b..7364e4b66 100644 --- a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts +++ b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts @@ -1,6 +1,6 @@ /** - * Exercise the CLI -> native staged DB -> durable checkpoint -> SIGKILL -> - * isolated recovery -> graph rebuild -> publication chain. The endpoint is + * Exercise a legacy native staged DB -> durable checkpoint -> SIGKILL -> + * current CLI recovery -> graph rebuild -> publication chain. The endpoint is * local and deterministic; request text is the billing/reuse oracle. */ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; @@ -16,11 +16,22 @@ const DIMS = 8; const NODE_COUNT = 5_128; const DEADLINE = process.env.CI ? 180_000 : 120_000; const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const adapterUrl = pathToFileURL(path.join(packageRoot, 'src/core/lbug/lbug-adapter.ts')).href; +const nativeUrl = pathToFileURL(path.join(packageRoot, 'src/core/lbug/lbug-config.ts')).href; +const analysisUrl = pathToFileURL( + path.join( + packageRoot, + process.env.GITNEXUS_E2E_CLI === 'dist' + ? 'dist/core/run-analyze.js' + : 'src/core/run-analyze.ts', + ), +).href; interface CheckpointMeta { repoPath: string; stats?: { embeddings?: number }; + embeddingJob?: unknown; + graphPath?: string; + vectorGeneration?: { identity: string; artifactIdentity?: string }; embeddingCheckpoint?: { nodesProcessed: number; chunksProcessed: number; @@ -37,11 +48,13 @@ interface Run { let root: string; let stoppedRepo: string; +let sourceAwareRepo: string; let server: http.Server; let endpoint: string; let submitted: string[] = []; let completed: string[] = []; let durableTexts: string[] = []; +let lastOutput = ''; const activeWindowCompleted: string[] = []; let held: string[] = []; let stopAfterCheckpoint = false; @@ -81,21 +94,41 @@ function cliEnv(repo: string): NodeJS.ProcessEnv { }; } -function runAnalyze(repo: string, flags: string[] = [], onOutput?: (output: string) => void): Run { +function runAnalyze( + repo: string, + flags: string[] = [], + onOutput?: (output: string) => void, + legacyFixture = false, +): Run { let output = ''; + // Generate the old checkpoint format through the retained legacy core path. + // Every recovery below still enters through the current public CLI. + const legacyScript = ` + const { runFullAnalysis } = await import(${JSON.stringify(analysisUrl)}); + await runFullAnalysis(${JSON.stringify(repo)}, { + force: true, embeddings: true, skipSkills: true, skipAgentsMd: true, + skipFts: true, workerPoolSize: 1, skipNativeCloseOnExit: true, + }, { onProgress() {}, onLog: console.log }); + process.exit(0); + `; + // A file entry keeps --input-type out of the parser workers' inherited argv. + const legacyEntry = path.join(root, 'legacy-analyze.mjs'); + if (legacyFixture) fs.writeFileSync(legacyEntry, legacyScript); const child = spawn( process.execPath, - [ - ...CLI_SPAWN_PREFIX, - 'analyze', - repo, - '--no-share', - '--skip-skills', - '--skip-fts', - '--workers', - '1', - ...flags, - ], + legacyFixture + ? ['--import', tsxLoaderUrl(), legacyEntry] + : [ + ...CLI_SPAWN_PREFIX, + 'analyze', + repo, + '--no-share', + '--skip-skills', + '--skip-fts', + '--workers', + '1', + ...flags, + ], { cwd: repo, env: cliEnv(repo), stdio: ['ignore', 'pipe', 'pipe'] }, ); running.add(child); @@ -124,6 +157,7 @@ async function successfulAnalyze(repo: string, flags: string[] = []): Promise row.nodeId)).size).toBe(expectedNodes); expect(meta.stats?.embeddings).toBe(rows.length); @@ -208,6 +258,53 @@ function expectPublishedComplete(repo: string, expectedNodes: number): void { ).toEqual([]); } +function expectLegacyReuse(texts: string[], unfinishedInputs: number): void { + // Legacy checkpoints record the provider but not chunk size/overlap. The + // complete single-chunk groups are provably reusable; the long group's + // historical input boundaries cannot be certified and must be explained. + const unverified = durableTexts.filter((text) => text.includes('longRecoverable')); + const reusable = new Set(durableTexts.filter((text) => !text.includes('longRecoverable'))); + expect(texts.filter((text) => reusable.has(text))).toEqual([]); + expect(unverified.length).toBeGreaterThan(1); + for (const text of unverified) expect(texts).toContain(text); + expect(texts).toHaveLength(unfinishedInputs + unverified.length); + expect(lastOutput).toContain('Legacy multi-chunk input provenance is unknown'); +} + +function removeLegacyInputProvenance(repo: string): void { + const stagingFile = readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile; + if (!stagingFile) throw new Error('fixture has no retained generation'); + // The retained core path now writes source-aware hashes. Downgrade only the + // long group's input digest to exercise checkpoints predating that receipt. + const script = ` + const native = await import(${JSON.stringify(nativeUrl)}); + const { default: lbug } = await import('@ladybugdb/core'); + const handle = await native.openLbugConnection(lbug, ${JSON.stringify(path.join(repo, '.gitnexus', stagingFile))}, { + bufferPoolBytes: 64 * 1024 ** 2, threads: 1, + }); + try { + const nodeId = 'Function:functions.ts:longRecoverable'; + const query = await handle.conn.prepare('MATCH (e:CodeEmbedding {nodeId: $nodeId}) RETURN e.contentHash AS contentHash'); + const result = await handle.conn.execute(query, { nodeId }); + const rows = await result.getAll(); + result.close(); + const hash = rows[0]?.contentHash; + if (rows.length < 2 || !/^[a-f0-9]{40}:[a-f0-9]{40}$/.test(hash) || rows.some(row => row.contentHash !== hash)) + throw new Error('Expected a complete source-aware multi-chunk fixture'); + const update = await handle.conn.prepare('MATCH (e:CodeEmbedding {nodeId: $nodeId}) SET e.contentHash = $contentHash'); + (await handle.conn.execute(update, { nodeId, contentHash: hash.slice(0, 40) })).close(); + console.log('LEGACY_INPUT_PROVENANCE_REMOVED'); + } finally { await native.closeLbugConnection(handle); } + `; + const result = spawnSync( + process.execPath, + ['--import', tsxLoaderUrl(), '--input-type=module', '-e', script], + { cwd: packageRoot, env: cliEnv(repo), encoding: 'utf8', timeout: 30_000 }, + ); + expect(result.status, `${result.error ?? ''}\n${result.stderr}\n${result.stdout}`).toBe(0); + expect(result.stdout).toContain('LEGACY_INPUT_PROVENANCE_REMOVED'); +} + beforeAll(async () => { if (process.platform === 'win32') return; root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-staged-recovery-e2e-')); @@ -219,7 +316,8 @@ beforeAll(async () => { { length: NODE_COUNT - 1 }, (_, index) => `export function recoverable${index}() { return ${index}; }`, ); - // Multi-chunk nodes must be reused as a complete group, including their tail. + // Missing legacy input provenance must invalidate the whole multi-chunk + // group, while retaining every provably compatible single-chunk group. const longFunction = `export function longRecoverable() {\n${Array.from({ length: 80 }, (_, index) => ` // retained chunk marker ${index} ${'x'.repeat(80)}`).join('\n')}\n return 42;\n}`; fs.writeFileSync( path.join(stoppedRepo, 'functions.ts'), @@ -284,7 +382,7 @@ beforeAll(async () => { const gate = new Promise((resolve) => { gateResolve = resolve; }); - const first = runAnalyze(stoppedRepo, ['--force', '--embeddings']); + const first = runAnalyze(stoppedRepo, [], undefined, true); await Promise.race([ gate, first.done.then((result) => { @@ -301,6 +399,8 @@ beforeAll(async () => { expect(completed.filter((text) => text.includes('longRecoverable')).length).toBeGreaterThan(1); expect(activeWindowCompleted).toHaveLength(64); expect(held.length).toBeGreaterThan(0); + sourceAwareRepo = cloneStoppedRepo('source-aware'); + removeLegacyInputProvenance(stoppedRepo); gateResolve = undefined; stopAfterCheckpoint = false; }, DEADLINE * 2); @@ -324,34 +424,44 @@ describe.skipIf(process.platform === 'win32')( 'interrupted staged embedding recovery (real CLI and native DB)', { concurrent: false }, () => { + it( + 'reuses verified source-aware multi-chunk groups without repeat provider work', + async () => { + const texts = await successfulAnalyze(sourceAwareRepo); + expect(texts.filter((text) => durableTexts.includes(text))).toEqual([]); + expect(texts).toHaveLength(128); + for (const text of held) expect(texts).toContain(text); + for (const text of activeWindowCompleted) expect(texts).toContain(text); + expectPublishedComplete(sourceAwareRepo, NODE_COUNT); + }, + DEADLINE, + ); + it.each([ ['plain', []], ['forced', ['--force', '--embeddings']], ] as const)( - '%s retry bills only unfinished groups and publishes an honest count', + '%s retry reuses verified groups and explains unverified legacy input', async (name, flags) => { const repo = cloneStoppedRepo(name); const texts = await successfulAnalyze(repo, [...flags]); - const durable = new Set(durableTexts); - expect(texts.filter((text) => durable.has(text))).toEqual([]); + expectLegacyReuse(texts, 128); for (const text of held) expect(texts).toContain(text); for (const text of activeWindowCompleted) expect(texts).toContain(text); - expect(texts.length).toBe(128); expectPublishedComplete(repo, NODE_COUNT); }, DEADLINE, ); it( - 'manual checkpoint opt-out completes a staged retry without resubmitting durable chunks', + 'manual checkpoint opt-out retains verified staged groups', async () => { const repo = cloneStoppedRepo('manual-checkpoint-opt-out'); const previous = process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = '0'; try { const texts = await successfulAnalyze(repo, ['--force', '--embeddings']); - expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(texts.length).toBe(128); + expectLegacyReuse(texts, 128); expectPublishedComplete(repo, NODE_COUNT); } finally { if (previous === undefined) delete process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; @@ -384,8 +494,7 @@ describe.skipIf(process.platform === 'win32')( expect(readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile).toBe(source); expect(fs.existsSync(path.join(repo, '.gitnexus', source))).toBe(true); const finalTexts = await successfulAnalyze(repo); - expect(finalTexts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(finalTexts.length).toBe(128); + expectLegacyReuse(finalTexts, 128); expectPublishedComplete(repo, NODE_COUNT); }, DEADLINE * 2, @@ -412,8 +521,7 @@ describe.skipIf(process.platform === 'win32')( true, ); expect(texts.some((text) => text.includes('recoverable6'))).toBe(false); - expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); - expect(texts.length).toBe(129); + expectLegacyReuse(texts, 129); expectPublishedComplete(repo, NODE_COUNT - 1); }, DEADLINE, diff --git a/gitnexus/test/integration/embedding-job-controller.test.ts b/gitnexus/test/integration/embedding-job-controller.test.ts new file mode 100644 index 000000000..e84efee6c --- /dev/null +++ b/gitnexus/test/integration/embedding-job-controller.test.ts @@ -0,0 +1,858 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { runPublishedEmbeddingJob } from '../../src/core/embeddings/embedding-job.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { EmbeddingSourceReader } from '../../src/core/embeddings/source-reader.js'; + +let temporary: TestDBHandle | undefined; +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await temporary?.cleanup(); + temporary = undefined; +}); + +async function fixture(count = 130) { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const root = temporary.dbPath; + let generation = 1; + const makeMeta = (): RepoMeta => ({ + repoPath: root, + lastCommit: 'commit', + indexedAt: new Date().toISOString(), + graphPath: `lbug.generation.${String(generation).padStart(32, '0')}`, + graphGeneration: { + version: 1, + id: String(generation).padStart(32, '0'), + coveredContent: 'a'.repeat(64), + }, + schemaFingerprint: 'graph-schema', + parseCacheIdentity: { base: 'parser' }, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'test', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: '127', + libc: 'glibc', + }, + invokedArtifact: { path: 'analyze', digest: 'cli' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'build', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'runtime', + }, + }, + scopeExtractionReceipt: 1, + }); + await saveMeta(root, makeMeta()); + const node = (id: string): EmbeddableNode => ({ + id, + name: id.split(':').at(-1) ?? id, + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, + }); + const scan = vi.fn(async function* (label: string) { + if (label !== 'Const') return; + for (let i = 0; i < count; i++) + yield { + id: `Const:a.ts:n${String(i).padStart(5, '0')}`, + contentBytes: Buffer.byteLength('const value = 1;'), + metadataBytes: 1024, + }; + }); + const fetch = vi.fn(async (_label: string, ids: readonly string[]) => ids.map(node)); + const close = vi.fn(async () => {}); + const openSource = vi.fn(async () => ({ scan, fetch, close })); + const initializeProvider = vi.fn(async () => {}); + const provider = vi.fn(async (texts: string[]) => { + expect((await loadMeta(root))?.embeddingJob).toBeDefined(); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }); + const identity = vi.fn(() => ({ provider: 'local', model: 'test', dimensions: 4 })); + const resources = { + graphPoolBytes: 16 * 1024 * 1024, + vectorPoolBytes: 64 * 1024 * 1024, + modelBytes: 0, + modelHeapMb: 0, + queueBytes: 16 * 1024 * 1024, + nativeThreads: 1, + reservedBytes: 96 * 1024 * 1024, + }; + const metadata = { + loadMetadata: () => loadMeta(root), + saveMetadata: (meta: RepoMeta) => saveMeta(root, meta), + }; + const options = () => ({ + metadataDirectory: root, + graphPath: path.join(root, `lbug.generation.${String(generation).padStart(32, '0')}`), + resources, + config: { threads: 1, subBatchSize: 8 }, + metadata, + }); + const dependencies = { openSource, provider, initializeProvider, identity }; + const run = (extra: Partial[0]> = {}) => + runPublishedEmbeddingJob({ ...options(), ...extra }, dependencies); + return { + root, + run, + options, + dependencies, + provider, + initializeProvider, + openSource, + scan, + fetch, + close, + nextGeneration: async () => { + const prior = await loadMeta(root); + generation++; + await saveMeta(root, { ...makeMeta(), vectorGeneration: prior?.vectorGeneration }); + }, + }; +} + +describe('pending artifacts across graph generations', () => { + it.each(['full', 'symbol', 'none'] as const)( + 'uses published repository source policy (%s), then resumes without source or provider work', + async (retention) => { + const f = await fixture(1); + const repo = path.join(f.root, 'checkout'); + await fs.mkdir(repo); + const content = `function sample() {\n${' consume("padding");\n'.repeat(300)} return "RECOVERED_TAIL";\n}`; + await fs.writeFile(path.join(repo, 'sample.ts'), content); + const node: EmbeddableNode = { + id: 'Function:sample.ts:sample', + label: 'Function', + name: 'sample', + filePath: 'sample.ts', + content: content.slice(0, 5000) + '... [truncated]', + startLine: 0, + endLine: content.split('\n').length - 1, + }; + f.scan.mockImplementation(async function* (label) { + if (label === 'Function') + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }); + f.fetch.mockImplementation(async () => [node]); + const meta = await loadMeta(f.root); + if (!meta) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...meta, + repoPath: repo, + contentRetention: retention, + fileHashes: { 'sample.ts': createHash('sha256').update(content).digest('hex') }, + }); + const read = vi.spyOn(EmbeddingSourceReader.prototype, 'read'); + const result = await f.run(); + const texts = f.provider.mock.calls.flatMap(([texts]) => texts).join('\n'); + expect(texts.includes('RECOVERED_TAIL')).toBe(retention === 'full'); + expect(read.mock.calls.length > 0).toBe(retention === 'full'); + expect(result.peakQueueBytes).toBeLessThanOrEqual(f.options().resources.queueBytes); + expect(node.content).toContain('[truncated]'); + if (retention === 'full') { + const advance = async (source: string) => { + await fs.writeFile(path.join(repo, 'sample.ts'), source); + await f.nextGeneration(); + const current = await loadMeta(f.root); + if (!current) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...current, + repoPath: repo, + contentRetention: retention, + fileHashes: { 'sample.ts': createHash('sha256').update(source).digest('hex') }, + }); + }; + f.provider.mockClear(); + read.mockClear(); + await advance(content + '\n// unrelated sibling edit'); + expect((await f.run()).state?.reusedNodes).toBe(1); + expect(f.provider).not.toHaveBeenCalled(); + expect(read).toHaveBeenCalled(); + await advance(content.replace('RECOVERED_TAIL', 'CHANGED_TAIL')); + await f.run(); + expect(f.provider.mock.calls.flatMap(([texts]) => texts).join('\n')).toContain( + 'CHANGED_TAIL', + ); + } + await fs.rm(repo, { recursive: true }); + f.provider.mockClear(); + read.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + }, + ); + + it('rejects oversized original source before paid work and retries with a larger reservation', async () => { + const f = await fixture(1); + const content = `function sample() {\n${' '.repeat(1024 * 1024)}return "TAIL";\n}`; + await fs.writeFile(path.join(f.root, 'sample.ts'), content); + const node: EmbeddableNode = { + id: 'Function:sample.ts:sample', + label: 'Function', + name: 'sample', + filePath: 'sample.ts', + content: 'function sample() ... [truncated]', + startLine: 0, + endLine: 2, + }; + f.scan.mockImplementation(async function* (label) { + if (label === 'Function') + yield { id: node.id, contentBytes: Buffer.byteLength(node.content), metadataBytes: 1024 }; + }); + f.fetch.mockImplementation(async () => [node]); + const meta = await loadMeta(f.root); + if (!meta) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { + ...meta, + fileHashes: { 'sample.ts': createHash('sha256').update(content).digest('hex') }, + }); + await expect(f.run()).rejects.toThrow('larger resource budget'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + const result = await f.run({ + resources: { ...f.options().resources, queueBytes: 32 * 1024 * 1024 }, + }); + expect(result.phase).toBe('ready'); + expect(result.peakQueueBytes).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(f.provider.mock.calls.flatMap(([texts]) => texts).join('\n')).toContain('TAIL'); + }); + + it('recovers native sealing before the ownership phase was recorded, then reuses read-only', async () => { + const f = await fixture(5); + const ownership = await import('../../src/storage/embedding-artifact.js'); + const adopt = ownership.adoptEmbeddingArtifact; + const interrupted = vi + .spyOn(ownership, 'adoptEmbeddingArtifact') + .mockImplementation(async (...args) => { + if (args[3].phase === 'sealed') throw new Error('stopped after native seal'); + return adopt(...args); + }); + await expect(f.run()).rejects.toThrow('stopped after native seal'); + interrupted.mockRestore(); + const first = (await loadMeta(f.root))?.embeddingJob; + await f.nextGeneration(); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(5); + expect(result.metadata.vectorGeneration?.artifactIdentity).not.toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('recovers an adopted artifact when metadata persistence fails before any new paid work', async () => { + const f = await fixture(130); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 64) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + metadata: { + ...f.options().metadata, + saveMetadata: async () => { + throw new Error('metadata unavailable'); + }, + }, + }), + ).rejects.toThrow('metadata unavailable'); + expect(f.provider).not.toHaveBeenCalled(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(result.state?.reusedNodes).toBe(64); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + }); + + it('retains the first paid groups across two interrupted rebases with a fresh cursor each time', async () => { + const f = await fixture(130); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 64) throw new Error('first interruption'); + }, + }), + ).rejects.toThrow('first interruption'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(64); + const first = (await loadMeta(f.root))?.embeddingJob; + if (!first?.artifactIdentity) throw new Error('Expected owned pending artifact'); + const file = path.join(f.root, 'embeddings', first.artifactIdentity, 'vectors'); + const inode = (await fs.stat(file)).ino; + const fetch = f.fetch.getMockImplementation(); + if (!fetch) throw new Error('Missing fixture fetch'); + const changed = new Set(['Const:a.ts:n00000']); + f.fetch.mockImplementation(async (label, ids) => + (await fetch(label, ids)).map((node) => + changed.has(node.id) ? { ...node, content: 'const value = 2;' } : node, + ), + ); + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes + state.reusedNodes === 128) + throw new Error('second interruption'); + }, + }), + ).rejects.toThrow('second interruption'); + const second = (await loadMeta(f.root))?.embeddingJob; + expect(second?.identity).not.toBe(first.identity); + expect(second?.artifactIdentity).toBe(first.artifactIdentity); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(65); + changed.add('Const:a.ts:n00001'); + await f.nextGeneration(); + f.provider.mockClear(); + const final = await f.run(); + expect(final.phase).toBe('ready'); + expect(final.state).toMatchObject({ completedNodes: 3, reusedNodes: 127, totalNodes: 130 }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + expect(final.metadata.vectorGeneration?.artifactIdentity).toBe(first.artifactIdentity); + expect(final.metadata.vectorGeneration?.graphGeneration).toBe( + final.metadata.graphGeneration?.id, + ); + expect((await fs.stat(file)).ino).toBe(inode); + const { searchVectorGeneration } = + await import('../../src/core/embeddings/vector-generation-search.js'); + const pool = await import('../../src/core/lbug/pool-adapter.js'); + try { + const hits = await searchVectorGeneration( + { metadataDirectory: f.root, metadata: final.metadata, graphPath: f.options().graphPath }, + async () => [1, 0, 0, 0], + 2, + 1, + async () => f.dependencies.identity(), + ); + expect(hits).toHaveLength(2); + expect(hits?.every((hit) => hit.nodeId.startsWith('Const:a.ts:'))).toBe(true); + } finally { + await pool.closeLbug(); + } + f.provider.mockClear(); + f.openSource.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('retains both pending paid groups and its completed fallback across another graph edit', async () => { + const f = await fixture(130); + const completed = await f.run(); + const fallback = completed.metadata.vectorGeneration; + if (!fallback?.artifactIdentity) throw new Error('Expected completed artifact'); + const fetch = f.fetch.getMockImplementation(); + if (!fetch) throw new Error('Missing fixture fetch'); + const changed = new Set(['Const:a.ts:n00000']); + f.fetch.mockImplementation(async (label, ids) => + (await fetch(label, ids)).map((node) => + changed.has(node.id) ? { ...node, content: 'const value = 2;' } : node, + ), + ); + await f.nextGeneration(); + f.provider.mockClear(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes + state.reusedNodes === 64) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + const pending = (await loadMeta(f.root))?.embeddingJob; + expect(pending?.artifactIdentity).not.toBe(fallback.artifactIdentity); + expect( + ( + await fs.stat(path.join(f.root, 'embeddings', fallback.artifactIdentity, 'vectors')) + ).isFile(), + ).toBe(true); + changed.add('Const:a.ts:n00001'); + await f.nextGeneration(); + f.provider.mockClear(); + const final = await f.run(); + expect(final.state).toMatchObject({ completedNodes: 1, reusedNodes: 129 }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(final.metadata.vectorGeneration?.artifactIdentity).toBe(pending?.artifactIdentity); + await expect( + fs.stat(path.join(f.root, 'embeddings', fallback.artifactIdentity, 'vectors')), + ).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('uses a sealed but unpublished artifact as a read-only fallback after a graph edit', async () => { + const f = await fixture(5); + const metadata = f.options().metadata; + await expect( + f.run({ + metadata: { + ...metadata, + saveMetadata: async (value) => { + if (value.vectorGeneration) throw new Error('publication interrupted'); + await metadata.saveMetadata(value); + }, + }, + }), + ).rejects.toThrow('publication interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + expect(first?.artifactIdentity).toBeDefined(); + await f.nextGeneration(); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(5); + expect(result.metadata.vectorGeneration?.artifactIdentity).not.toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('returns directly to retained compatible work after using another vector space', async () => { + const f = await fixture(10); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes === 10) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const first = (await loadMeta(f.root))?.embeddingJob; + f.dependencies.identity.mockReturnValue({ provider: 'local', model: 'other', dimensions: 4 }); + await f.run(); + f.dependencies.identity.mockReturnValue({ provider: 'local', model: 'test', dimensions: 4 }); + f.provider.mockClear(); + f.openSource.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('fails before source/model work when the directly keyed ownership receipt is corrupt', async () => { + const f = await fixture(5); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedNodes) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const pending = (await loadMeta(f.root))?.embeddingJob; + if (!pending?.spaceKey || !pending.artifactIdentity) + throw new Error('Missing ownership receipt'); + const receipt = path.join(f.root, 'embeddings', 'spaces', pending.spaceKey, 'space.json'); + await fs.writeFile(receipt, '{"checksum":"bad","payload":{}}'); + f.provider.mockClear(); + f.openSource.mockClear(); + f.initializeProvider.mockClear(); + await expect(f.run()).rejects.toThrow('checksum mismatch'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + await expect( + fs.stat(path.join(f.root, 'embeddings', pending.artifactIdentity, 'vectors')), + ).resolves.toBeDefined(); + }); +}); + +describe('published graph embedding job controller', () => { + it('reclaims the prior rebased artifact after a third generation seals', async () => { + const f = await fixture(3); + await f.run(); + await f.nextGeneration(); + const second = await f.run(); + const previous = second.metadata.vectorGeneration!; + expect(previous.artifactIdentity).not.toBe(previous.identity); + const oldDirectory = path.join(f.root, 'embeddings', previous.artifactIdentity!); + expect( + JSON.parse(await fs.readFile(path.join(oldDirectory, 'artifact.json'), 'utf8')).payload + .identity, + ).toBe(previous.artifactIdentity); + await f.nextGeneration(); + const third = await f.run(); + expect(third.phase).toBe('ready'); + await expect(fs.stat(oldDirectory)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + it('resumes an interrupted bootstrap without rescanning the manifest or losing the pending job', async () => { + const f = await fixture(3); + const original = VectorStore.open.bind(VectorStore); + const open = vi.spyOn(VectorStore, 'open').mockImplementationOnce(async (options) => { + await expect( + promisify(execFile)(process.execPath, [ + fileURLToPath(new URL('../helpers/interrupted-vector-bootstrap.mjs', import.meta.url)), + JSON.stringify(options), + ]), + ).rejects.toMatchObject({ code: 73 }); + throw new Error('interrupted bootstrap'); + }); + await expect(f.run()).rejects.toThrow('interrupted bootstrap'); + expect(f.provider).not.toHaveBeenCalled(); + const pending = (await loadMeta(f.root))?.embeddingJob; + expect(pending).toBeDefined(); + f.scan.mockClear(); + open.mockImplementation(original); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.identity).toBe(pending?.identity); + expect(f.scan).not.toHaveBeenCalled(); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + }); + it('drains an earlier local runtime before admitting an HTTP job with no model reservation', async () => { + const f = await fixture(3); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'https://embedding.test/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'test-model'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + const sidecar = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const drain = vi.spyOn(sidecar, 'drainEmbeddingSidecarForAnalysis').mockResolvedValue(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { input: string[] }; + return new Response( + JSON.stringify({ + data: body.input.map((_text, index) => ({ index, embedding: [1, 0, 0, 0] })), + }), + ); + }); + const result = await runPublishedEmbeddingJob(f.options(), { openSource: f.openSource }); + expect(result.phase).toBe('ready'); + expect(drain).toHaveBeenCalledTimes(1); + expect(drain.mock.invocationCallOrder[0]).toBeLessThan( + f.openSource.mock.invocationCallOrder[0], + ); + }); + + it('uses and releases the admitted local runtime, including initialization failure', async () => { + const f = await fixture(3); + const sidecar = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const initialize = vi.fn(async () => {}); + const embedBatch = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([1, 0, 0, 0])), + ); + const release = vi.fn(); + const reserve = vi + .spyOn(sidecar, 'reserveEmbeddingSidecar') + .mockResolvedValue({ initialize, embedBatch, release }); + const options = { + ...f.options(), + resources: { ...f.options().resources, modelBytes: 1024 * 1024 * 1024, modelHeapMb: 256 }, + }; + const dependencies = { identity: f.dependencies.identity, openSource: f.openSource }; + await expect( + runPublishedEmbeddingJob( + { ...options, resources: { ...options.resources, modelBytes: NaN } }, + dependencies, + ), + ).rejects.toThrow('resource reservation'); + expect(reserve).not.toHaveBeenCalled(); + expect((await loadMeta(f.root))?.embeddingJob).toBeUndefined(); + initialize.mockRejectedValueOnce(new Error('model init unavailable')); + await expect(runPublishedEmbeddingJob(options, dependencies)).rejects.toThrow( + 'model init unavailable', + ); + expect(embedBatch).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledTimes(1); + expect((await loadMeta(f.root))?.embeddingJob).toBeDefined(); + const result = await runPublishedEmbeddingJob(options, dependencies); + expect(result.phase).toBe('ready'); + expect(reserve).toHaveBeenCalledWith(expect.objectContaining({ threads: 1 }), 256); + expect(release).toHaveBeenCalledTimes(2); + expect(embedBatch.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + reserve.mockClear(); + expect((await runPublishedEmbeddingJob(options, dependencies)).alreadyComplete).toBe(true); + expect(reserve).not.toHaveBeenCalled(); + }); + + it('rejects a split vector pool below the native minimum before changing metadata', async () => { + const f = await fixture(3); + await f.run(); + await f.nextGeneration(); + const before = await loadMeta(f.root); + f.openSource.mockClear(); + await expect( + f.run({ resources: { ...f.options().resources, vectorPoolBytes: 8 * 1024 * 1024 } }), + ).rejects.toThrow('reuse reader do not fit'); + expect(await loadMeta(f.root)).toEqual(before); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('publishes complete vectors, then takes a no-model/no-writer fast path', async () => { + const f = await fixture(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.nodes).toBe(130); + expect(result.metadata.embeddingJob).toBeUndefined(); + expect(result.state?.completedUnits).toBe(3); + expect(f.initializeProvider).toHaveBeenCalledOnce(); + f.provider.mockClear(); + f.openSource.mockClear(); + const open = vi.spyOn(VectorStore, 'open'); + expect((await f.run()).alreadyComplete).toBe(true); + expect(open).not.toHaveBeenCalled(); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + }); + + it('resumes acknowledged work with a smaller batch and queue without rescanning keys', async () => { + const f = await fixture(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits === 1) throw new Error('stop after acknowledgement'); + }, + }), + ).rejects.toThrow('stop after acknowledgement'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(64); + f.provider.mockClear(); + f.scan.mockClear(); + const result = await f.run({ + config: { threads: 1, subBatchSize: 1 }, + resources: { ...f.options().resources, queueBytes: 8 * 1024 * 1024 }, + }); + expect(result.phase).toBe('ready'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + expect(f.scan).not.toHaveBeenCalled(); + expect(result.peakQueueBytes).toBeLessThanOrEqual(8 * 1024 * 1024); + }); + + it('retries pointer publication without source or provider work', async () => { + const f = await fixture(5); + const metadata = f.options().metadata; + await expect( + f.run({ + metadata: { + ...metadata, + saveMetadata: async (meta) => { + if (meta.vectorGeneration) throw new Error('disk unavailable for pointer'); + await metadata.saveMetadata(meta); + }, + }, + }), + ).rejects.toThrow('disk unavailable'); + f.provider.mockClear(); + f.openSource.mockClear(); + f.initializeProvider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + }); + + it('copies compatible paid vectors to the next graph with no provider call', async () => { + const f = await fixture(5); + const first = await f.run(); + await f.nextGeneration(); + f.provider.mockClear(); + f.initializeProvider.mockClear(); + const next = await f.run(); + expect(next.phase).toBe('ready'); + expect(next.metadata.vectorGeneration?.identity).not.toBe( + first.metadata.vectorGeneration?.identity, + ); + expect(next.state?.reusedNodes).toBe(5); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + if (!first.metadata.vectorGeneration) throw new Error('Expected published vectors'); + await expect( + fs.stat(path.join(f.root, 'embeddings', first.metadata.vectorGeneration.identity, 'vectors')), + ).rejects.toMatchObject({ code: 'ENOENT' }); + expect( + ( + await fs.stat( + path.join( + f.root, + 'embeddings', + next.metadata.vectorGeneration!.artifactIdentity!, + 'vectors', + ), + ) + ).isFile(), + ).toBe(true); + }); + + it('does no paid replay if the process stops after publishing the vector pointer', async () => { + const f = await fixture(5); + const ready = vi + .spyOn(EmbeddingJobJournal.prototype, 'ready') + .mockRejectedValueOnce(new Error('stopped after pointer')); + await expect(f.run()).rejects.toThrow('stopped after pointer'); + expect((await loadMeta(f.root))?.vectorGeneration).toBeDefined(); + f.provider.mockClear(); + f.openSource.mockClear(); + expect((await f.run()).alreadyComplete).toBe(true); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.openSource).not.toHaveBeenCalled(); + ready.mockRestore(); + }); + + it('recovers an interrupted manifest preflight before any provider or vector mutation', async () => { + const f = await fixture(5); + f.scan.mockImplementationOnce(async function* () { + throw new Error('preflight stopped'); + }); + await expect(f.run()).rejects.toThrow('preflight stopped'); + expect(f.provider).not.toHaveBeenCalled(); + expect((await f.run()).phase).toBe('ready'); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(5); + }); + + it('publishes empty coverage without a model request', async () => { + const f = await fixture(0); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.metadata.vectorGeneration?.chunks).toBe(0); + expect(result.metadata.capabilities?.vectorSearch.status).toBe('unavailable'); + expect(f.provider).not.toHaveBeenCalled(); + }); + + it('admits a 10 MiB fallback file and embeds its final marker without truncation', async () => { + const f = await fixture(0); + const content = 'a'.repeat(10 * 1024 * 1024 - 30) + '\nFINAL_UNTRUNCATED_MARKER'; + const id = 'File:large.txt'; + let sawEnd = false; + let requests = 0; + const result = await runPublishedEmbeddingJob( + { + ...f.options(), + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 16 * 1024 ** 3, + availableCpus: 4, + totalMb: 8192, + modelMemoryMb: 0, + includeParsing: false, + }).embedding, + }, + { + ...f.dependencies, + openSource: async () => ({ + async *scan(label) { + if (label === 'File') + yield { id, contentBytes: Buffer.byteLength(content), metadataBytes: 1024 }; + }, + fetch: async () => [ + { + id, + label: 'File', + name: 'large.txt', + filePath: 'large.txt', + content, + startLine: 1, + endLine: 2, + }, + ], + close: async () => {}, + }), + // Count inputs without retaining the complete test corpus in spy history. + provider: async (texts) => { + requests++; + sawEnd ||= texts.some((text) => text.includes('FINAL_UNTRUNCATED_MARKER')); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }, + }, + ); + expect(result.phase).toBe('ready'); + expect(sawEnd).toBe(true); + expect(requests).toBeGreaterThan(100); + expect(result.peakQueueBytes).toBeLessThanOrEqual(256 * 1024 * 1024); + }, 60000); + + it('reconstructs a corrupt journal only from verified native complete groups', async () => { + const f = await fixture(); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits === 1) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const meta = await loadMeta(f.root); + if (!meta?.embeddingJob) throw new Error('Expected pending job'); + const job = path.join(f.root, 'embeddings', meta.embeddingJob.identity); + await fs.writeFile(path.join(job, 'job.json'), '{broken'); + f.provider.mockClear(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(64); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(66); + expect((await fs.readdir(job)).some((name) => name.startsWith('job.json.preserved-'))).toBe( + true, + ); + }); + + it('rejects a missing paid artifact before model or fresh-store creation', async () => { + const f = await fixture(5); + await expect( + f.run({ + onProgress: (state) => { + if (state.completedUnits) throw new Error('interrupted'); + }, + }), + ).rejects.toThrow('interrupted'); + const meta = await loadMeta(f.root); + if (!meta?.embeddingJob) throw new Error('Expected pending job'); + const job = path.join(f.root, 'embeddings', meta.embeddingJob.identity); + await fs.rename(path.join(job, 'vectors'), path.join(job, 'vectors.retained')); + f.provider.mockClear(); + await expect(f.run()).rejects.toThrow('vectors are missing'); + expect(f.provider).not.toHaveBeenCalled(); + await expect(fs.stat(path.join(job, 'vectors'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('binds graph paths through an aliased ancestor to the canonical job root', async () => { + const f = await fixture(1); + const alias = path.join(f.root, 'alias'); + await fs.symlink( + path.dirname(f.root), + alias, + process.platform === 'win32' ? 'junction' : 'dir', + ); + const graphFile = path.basename(f.options().graphPath); + const result = await f.run({ + graphPath: path.join(alias, path.basename(f.root), graphFile), + }); + expect(result.phase).toBe('ready'); + expect(f.openSource).toHaveBeenCalledWith( + path.join(await fs.realpath(f.root), graphFile), + f.options().resources.graphPoolBytes, + f.options().resources.nativeThreads, + ); + }); + + it('refuses a symlink job root before writing metadata or requesting vectors', async () => { + const f = await fixture(5); + await fs.mkdir(path.join(f.root, 'other')); + await fs.symlink(path.join(f.root, 'other'), path.join(f.root, 'embeddings'), 'dir'); + await expect(f.run()).rejects.toThrow('owned directory'); + expect((await loadMeta(f.root))?.embeddingJob).toBeUndefined(); + expect(f.provider).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/integration/embedding-job-restart.test.ts b/gitnexus/test/integration/embedding-job-restart.test.ts new file mode 100644 index 000000000..6086a6d7d --- /dev/null +++ b/gitnexus/test/integration/embedding-job-restart.test.ts @@ -0,0 +1,159 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; + +const execute = promisify(execFile); +const helper = fileURLToPath(new URL('../helpers/embedding-restart-runner.mjs', import.meta.url)); +const boundaries = [ + 'intent-before', + 'intent-after', + 'appendBatch-before', + 'appendBatch-after', + 'checkpointUnit-before', + 'checkpointUnit-after', + 'cursor-before', + 'cursor-after', + 'seal-after', + 'pointer-after', +] as const; + +it.each(boundaries)( + 'resumes a real process exit at %s without replaying acknowledged inputs', + async (boundary) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-restart-')); + const env = { + ...process.env, + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + }; + let store: VectorStore | undefined; + try { + await saveMeta(root, { + repoPath: root, + lastCommit: 'fixture', + indexedAt: 'fixture', + graphPath: `lbug.generation.${'a'.repeat(32)}`, + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + schemaFingerprint: 'fixture', + parseCacheIdentity: { base: 'fixture' }, + scopeExtractionReceipt: 1, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'fixture', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: process.versions.modules, + libc: 'fixture', + }, + invokedArtifact: { path: helper, digest: 'fixture' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'fixture', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'fixture', + }, + }, + }); + const stopped = await execute(process.execPath, [helper, root, boundary], { env }).then( + () => ({ code: 0, stderr: '' }), + (error: NodeJS.ErrnoException & { stderr?: string }) => error, + ); + expect(stopped.code, stopped.stderr).toBe(73); + await execute(process.execPath, [helper, root, 'resume'], { env }); + const readInputs = async ( + phase: string, + ): Promise> => { + const text = await fs + .readFile(path.join(root, `${phase}.inputs.jsonl`), 'utf8') + .catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return ''; + throw error; + }); + return text + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + }; + const first = await readInputs(boundary); + const resumed = await readInputs('resume'); + expect(resumed.every(({ node }) => node === null || Number(node.slice(1)) >= 64)).toBe(true); + if (boundary === 'seal-after' || boundary === 'pointer-after') expect(resumed).toEqual([]); + const paid = new Map(); + for (const input of [...first, ...resumed]) { + const entry = paid.get(input.hash) ?? { node: input.node, count: 0 }; + entry.count++; + paid.set(input.hash, entry); + } + expect( + new Set([...paid.values()].map(({ node }) => node).filter((node) => node !== null)), + ).toEqual( + new Set(Array.from({ length: 130 }, (_, index) => `n${String(index).padStart(5, '0')}`)), + ); + for (const { node, count } of paid.values()) { + // Only n00064 has interior slices without a name, all in the active unit. + const mayReplay = + node === null || (Number(node.slice(1)) >= 64 && Number(node.slice(1)) < 128); + expect(count).toBeLessThanOrEqual(mayReplay ? 2 : 1); + } + const metadata = await loadMeta(root); + const generation = metadata?.vectorGeneration; + if (!generation) throw new Error('Restart did not publish vectors'); + expect(metadata.embeddingJob).toBeUndefined(); + expect(generation.nodes).toBe(130); + store = await VectorStore.open({ + directory: path.join( + root, + 'embeddings', + generation.artifactIdentity ?? generation.identity, + ), + identity: generation.artifactIdentity ?? generation.identity, + dimensions: 4, + mode: 'read', + bufferPoolBytes: 64 * 1024 ** 2, + threads: 1, + batchBytes: 64 * 1024, + }); + let totalChunks = 0; + for (let offset = 0; offset < 130; offset += 64) { + const ids = Array.from( + { length: Math.min(64, 130 - offset) }, + (_, index) => `Function:a.ts:n${String(offset + index).padStart(5, '0')}`, + ); + const complete = await store.completedNodes(ids); + expect(complete.size).toBe(ids.length); + for (const [id, receipt] of complete) { + const chunks = await store.readChunks(receipt, 0, receipt.chunks); + expect(chunks.map(({ chunkIndex }) => chunkIndex)).toEqual( + Array.from({ length: receipt.chunks }, (_, index) => index), + ); + if (id.endsWith('n00064')) expect(chunks.length).toBeGreaterThan(1); + totalChunks += chunks.length; + } + } + expect(totalChunks).toBe(generation.chunks); + } finally { + await store?.close(); + await fs.rm(root, { recursive: true, force: true }); + } + }, + 120_000, +); diff --git a/gitnexus/test/integration/embedding-native-release.test.ts b/gitnexus/test/integration/embedding-native-release.test.ts new file mode 100644 index 000000000..3110ed954 --- /dev/null +++ b/gitnexus/test/integration/embedding-native-release.test.ts @@ -0,0 +1,139 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import { + closeLbugConnection, + drainLbugResourceReleases, + openLbugConnection, +} from '../../src/core/lbug/lbug-config.js'; + +describe('embedding native handle ownership', () => { + let root: string; + const opened: Array<{ close(): Promise }> = []; + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-native-release-')); + }); + afterEach(async () => { + vi.restoreAllMocks(); + for (const handle of opened.splice(0)) await handle.close(); + await drainLbugResourceReleases(); + await fs.rm(root, { recursive: true, force: true }); + }); + + async function open(kind: 'graph' | 'vector') { + if (kind === 'vector') { + const store = await VectorStore.open({ + directory: root, + identity: 'a'.repeat(64), + dimensions: 2, + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + batchBytes: 1024, + mode: 'create', + }); + opened.push(store); + return store; + } + const graphPath = path.join(root, 'graph'); + const writer = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + }); + try { + for (const sql of [ + 'CREATE NODE TABLE File (id STRING, name STRING, filePath STRING, content STRING, PRIMARY KEY(id))', + "CREATE (:File {id:'File:one.ts',name:'one.ts',filePath:'one.ts',content:'export const one = 1;'})", + 'CHECKPOINT', + ]) { + const result = await writer.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } + } finally { + await closeLbugConnection(writer); + } + const source = await EmbeddingGraphSource.open(graphPath, 16 * 1024 * 1024, 1); + opened.push(source); + let count = 0; + for await (const _ of source.scan('File')) count++; + expect(count).toBe(1); + return source; + } + + it.each(['scan', 'fetch'] as const)( + 'allows graph teardown after a %s result fails to close', + async (operation) => { + const source = (await open('graph')) as EmbeddingGraphSource; + const original = lbug.QueryResult.prototype.close; + vi.spyOn(lbug.QueryResult.prototype, 'close').mockImplementationOnce(function ( + this: lbug.QueryResult, + ) { + original.call(this); + throw new Error('result release failed'); + }); + const read = + operation === 'fetch' + ? source.fetch('File', ['File:one.ts']) + : (async () => { + for await (const _ of source.scan('File')) { + /* drain */ + } + })(); + await expect(read).rejects.toThrow('result release failed'); + await expect(source.close()).resolves.toBeUndefined(); + }, + ); + + it.each([ + ['graph', 'connection'], + ['graph', 'database'], + ['vector', 'connection'], + ['vector', 'database'], + ] as const)('retries a failed %s %s close before claiming release', async (kind, part) => { + const handle = await open(kind); + const prototype = part === 'connection' ? lbug.Connection.prototype : lbug.Database.prototype; + const close = vi + .spyOn(prototype, 'close') + .mockRejectedValueOnce(new Error('native release failed')); + await expect(handle.close()).rejects.toThrow('native release failed'); + await handle.close(); + expect(close).toHaveBeenCalledTimes(2); + await handle.close(); + expect(close).toHaveBeenCalledTimes(2); + }); + + it.each(['graph', 'vector'] as const)( + 'keeps concurrent %s close callers pending until native exit', + async (kind) => { + const handle = await open(kind); + const original = lbug.Connection.prototype.close; + let release!: () => void; + const wait = new Promise((resolve) => { + release = resolve; + }); + const close = vi + .spyOn(lbug.Connection.prototype, 'close') + .mockImplementationOnce(async function (this: lbug.Connection) { + await wait; + await original.call(this); + }); + const first = handle.close(); + let finished = false; + const second = handle.close().then(() => { + finished = true; + }); + try { + await Promise.resolve(); + expect(finished).toBe(false); + expect(close).toHaveBeenCalledTimes(1); + } finally { + release(); + await Promise.all([first, second]); + } + expect(finished).toBe(true); + }, + ); +}); diff --git a/gitnexus/test/integration/embedding-vector-store.test.ts b/gitnexus/test/integration/embedding-vector-store.test.ts new file mode 100644 index 000000000..26f58ef5f --- /dev/null +++ b/gitnexus/test/integration/embedding-vector-store.test.ts @@ -0,0 +1,1592 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type VectorStoreOptions } from '../../src/core/lbug/vector-store.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; +import { EmbeddingGraphSource } from '../../src/core/lbug/embedding-source.js'; +import { runEmbeddingJob } from '../../src/core/embeddings/embedding-job-runner.js'; +import { + createEmbeddingWorkManifest, + loadEmbeddingWorkManifest, +} from '../../src/core/embeddings/embedding-work-manifest.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { HttpEmbeddingError } from '../../src/core/embeddings/http-client.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { createHash, randomUUID } from 'node:crypto'; +import { searchVectorGeneration } from '../../src/core/embeddings/vector-generation-search.js'; +import * as readerPool from '../../src/core/lbug/pool-adapter.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { saveMeta, loadMeta } from '../../src/storage/repo-manager.js'; +import { publishVectorGeneration } from '../../src/core/embeddings/vector-publication.js'; +import { + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { ExtensionManager } from '../../src/core/lbug/extension-loader.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +let temporary: TestDBHandle | undefined; +let store: VectorStore; +const chunk = (chunkIndex: number) => ({ + chunkIndex, + startLine: 1, + endLine: 2, + embedding: [0.1, 0.2, 0.3, 0.4], +}); +async function setup() { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const options: VectorStoreOptions = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'create', + }; + store = await VectorStore.open(options); + return options; +} +afterEach(async () => { + vi.restoreAllMocks(); + await store?.close(); + store = undefined; + await temporary?.cleanup(); + temporary = undefined; + vi.unstubAllEnvs(); +}); + +async function runnerFixture(count: number, content = 'const value = "漢😀";') { + const options = await setup(); + const node = (id: string): EmbeddableNode => ({ + id, + name: id.split(':').at(-1), + label: 'Const', + filePath: 'source.ts', + content, + startLine: 1, + endLine: 1, + }); + const fetch = vi.fn(async (_label: string, ids: readonly string[]) => ids.map(node)); + const source = { + async *scan(label: string) { + if (label !== 'Const') return; + for (let i = 0; i < count; i++) + yield { + id: `Const:source.ts:n${String(i).padStart(5, '0')}`, + contentBytes: Buffer.byteLength(content), + metadataBytes: 1024, + }; + }, + fetch, + }; + const manifest = await createEmbeddingWorkManifest(options.directory, 'source', source); + let journal = await EmbeddingJobJournal.create(options.directory, options.identity, manifest); + const initializeProvider = vi.fn(async () => {}); + const provider = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + const run = (extra = {}) => + runEmbeddingJob({ + directory: options.directory, + manifest, + source, + journal, + store, + queueBytes: 32 * 1024 * 1024, + config: { subBatchSize: 8, maxInputChars: 1024 }, + provider, + initializeProvider, + ...extra, + }); + const reopen = async (recover = false) => { + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + const selection = await loadEmbeddingWorkManifest(options.directory, 'source'); + journal = await (recover ? EmbeddingJobJournal.recover : EmbeddingJobJournal.open).call( + EmbeddingJobJournal, + options.directory, + options.identity, + selection, + ); + }; + return { + options, + source, + manifest, + get journal() { + return journal; + }, + initializeProvider, + provider, + run, + reopen, + }; +} + +describe('rebased pending vector inputs', () => { + it('reuses matching groups in place, relocates lines, recomputes changed inputs and prunes deleted nodes', async () => { + const f = await runnerFixture(2); + const { contentHashForNode } = await import('../../src/core/embeddings/embedding-pipeline.js'); + const { resolveEmbeddingConfig } = await import('../../src/core/embeddings/config.js'); + const ids = ['Const:source.ts:n00000', 'Const:source.ts:n00001', 'Const:source.ts:deleted']; + const nodes = await f.source.fetch('Const', ids); + const hash = contentHashForNode( + nodes[0], + resolveEmbeddingConfig({ subBatchSize: 8, maxInputChars: 1024 }), + ); + for (const [i, id] of ids.entries()) { + const inputHash = i === 0 ? hash : 'previous-input'; + await store.beginNodes([ + { nodeId: id, contentHash: inputHash, sourceStartLine: 9, sourceEndLine: 9 }, + ]); + await store.append(id, inputHash, [{ ...chunk(0), startLine: 9, endLine: 9 }]); + const receipt = await store.finishNode(id, inputHash, 1); + await store.checkpointUnit([receipt]); + } + const inode = (await fs.stat(path.join(f.options.directory, 'vectors'))).ino; + const result = await f.run({ revalidateStoredInputs: true }); + expect(result.state).toMatchObject({ + completedNodes: 1, + reusedNodes: 1, + failedNodes: 0, + chunks: 2, + }); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(await store.reusable(ids[2], 'previous-input')).toBeNull(); + const moved = await store.reusable(ids[0], hash); + expect(moved).toMatchObject({ sourceStartLine: 1, sourceEndLine: 1 }); + if (!moved) throw new Error('Matching input was not retained'); + expect(await store.readChunks(moved, 0, 1)).toMatchObject([{ startLine: 1, endLine: 1 }]); + expect((await fs.stat(path.join(f.options.directory, 'vectors'))).ino).toBe(inode); + }); + + it('refuses to prune when current-selection coverage is missing', async () => { + await setup(); + const id = 'Const:source.ts:retained'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await expect(store.pruneUnselected('b'.repeat(64), 1)).rejects.toThrow('no groups pruned'); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + await store.markSelected('b'.repeat(64), [id]); + await expect(store.pruneUnselected('c'.repeat(64), 1)).rejects.toThrow('no groups pruned'); + await store.pruneUnselected('b'.repeat(64), 1); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + }); + + it('prunes an empty new selection and leaves a valid empty artifact', async () => { + await setup(); + const id = 'Const:source.ts:deleted'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.markSelected('b'.repeat(64), [id]); + await store.pruneUnselected('c'.repeat(64), 0); + expect(await store.reusable(id, 'hash')).toBeNull(); + expect(await store.seal(0, 0)).toBe('exact'); + }); + + it('keeps sealed artifacts immutable through every rebase operation', async () => { + await setup(); + const id = 'Const:source.ts:retained'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.seal(1, 1); + expect(store.isSealed).toBe(true); + await expect(store.reuseNode({ nodeId: id, contentHash: 'hash' })).rejects.toThrow('immutable'); + await expect(store.markSelected('b'.repeat(64), [id])).rejects.toThrow('immutable'); + await expect(store.pruneUnselected('b'.repeat(64), 0)).rejects.toThrow('immutable'); + expect(await store.reusable(id, 'hash')).not.toBeNull(); + }); + + it('relocates a multi-batch group without copying or changing its vectors', async () => { + await setup(); + const id = 'Function:source.ts:large'; + await store.beginNodes([ + { nodeId: id, contentHash: 'hash', sourceStartLine: 10, sourceEndLine: 309 }, + ]); + for (let start = 0; start < 300; start += 100) { + await store.append( + id, + 'hash', + Array.from({ length: 100 }, (_, i) => ({ + ...chunk(start + i), + startLine: start + i + 10, + endLine: start + i + 10, + })), + ); + } + await store.finishNode(id, 'hash', 300); + expect(await store.reuseNode({ nodeId: id, contentHash: 'changed' })).toBeNull(); + const receipt = await store.reuseNode({ + nodeId: id, + contentHash: 'hash', + sourceStartLine: 15, + sourceEndLine: 314, + }); + expect(receipt?.chunks).toBe(300); + if (!receipt) throw new Error('Complete group was not retained'); + const first = (await store.readChunks(receipt, 0, 1))[0]; + const last = (await store.readChunks(receipt, 299, 1))[0]; + expect(first).toMatchObject({ startLine: 15, endLine: 15 }); + expect(last).toMatchObject({ startLine: 314, endLine: 314 }); + expect(first.embedding).toEqual(last.embedding); + expect(first.embedding[0]).toBeCloseTo(0.1); + await store.markSelected('b'.repeat(64), [id]); + await store.pruneUnselected('b'.repeat(64), 1); + await store.seal(1, 300); + }); +}); + +describe('bounded durable embedding runner', () => { + it.each([256, 512])( + 'retains generated writes within a %i KiB queue until durable completion', + async (queueKiB) => { + const fixture = await runnerFixture(64); + const append = vi.spyOn(store, 'appendBatch'); + const result = await fixture.run({ queueBytes: queueKiB * 1024 }); + expect(result.state).toMatchObject({ completedNodes: 64, chunks: 64, failedUnits: 0 }); + expect(result.peakQueueBytes).toBeLessThanOrEqual(queueKiB * 1024); + expect(append.mock.calls.flatMap(([rows]) => rows)).toHaveLength(64); + if (queueKiB === 512) + expect(append.mock.calls.length).toBeLessThan(fixture.provider.mock.calls.length); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).state.chunks).toBe(64); + expect(fixture.provider).not.toHaveBeenCalled(); + }, + ); + + it('completes copied groups together at the unit boundary without provider requests', async () => { + const fixture = await runnerFixture(64); + const finish = vi.spyOn(store, 'finishNodes'); + const queries = vi.spyOn(lbug.Connection.prototype, 'query'); + const result = await fixture.run({ + queueBytes: 512 * 1024, + reuse: { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 }; + }, + async readChunks() { + return [chunk(0)]; + }, + }, + }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(result.state).toMatchObject({ reusedNodes: 64, completedNodes: 0, chunks: 64 }); + expect(finish).toHaveBeenCalledTimes(1); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + // A complete unit must not pay one durable native commit per reused node. + expect(queries.mock.calls.filter(([sql]) => sql === 'COMMIT').length).toBeLessThan(10); + }); + + it('bounds copied and generated vectors together and preserves complete mixed coverage', async () => { + const fixture = await runnerFixture(65); + const result = await fixture.run({ + queueBytes: 512 * 1024, + reuse: { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return Number(nodeId.slice(-5)) < 33 + ? { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 } + : null; + }, + async readChunks() { + return [chunk(0)]; + }, + }, + }); + expect(result.state).toMatchObject({ reusedNodes: 33, completedNodes: 32, chunks: 65 }); + const submitted = fixture.provider.mock.calls.flatMap(([texts]) => texts); + expect(submitted).toHaveLength(32); + expect(new Set(submitted).size).toBe(32); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + }); + + it.each(['read', 'write', 'cancel'] as const)( + 'replays a copy %s interruption without duplicate chunks or provider requests', + async (fault) => { + const fixture = await runnerFixture(64); + let fail = true; + const controller = new AbortController(); + const append = store.appendBatch.bind(store); + vi.spyOn(store, 'appendBatch').mockImplementation(async (chunks) => { + await append(chunks); + if (fail && fault === 'write') throw new Error('injected reuse failure'); + }); + const reuse = { + options: { batchBytes: 64 * 1024 }, + async reusable(nodeId: string, contentHash: string) { + return { nodeId, contentHash, chunks: 1, complete: true, sourceStartLine: 1 }; + }, + async readChunks(receipt: { nodeId: string }) { + if (fail && receipt.nodeId.endsWith('n00055')) { + if (fault === 'read') throw new Error('injected reuse failure'); + if (fault === 'cancel') controller.abort(new Error('injected reuse failure')); + } + return [chunk(0)]; + }, + }; + await expect( + fixture.run({ reuse, queueBytes: 512 * 1024, signal: controller.signal }), + ).rejects.toThrow('reuse failure'); + expect(fixture.journal.snapshot()).toMatchObject({ reusedNodes: 0, chunks: 0 }); + expect(fixture.journal.snapshot().active).not.toBeNull(); + await fixture.reopen(); + fail = false; + const result = await fixture.run({ reuse, queueBytes: 512 * 1024 }); + expect(result.state).toMatchObject({ reusedNodes: 64, completedNodes: 0, chunks: 64 }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(result.peakQueueBytes).toBeLessThanOrEqual(512 * 1024); + }, + ); + + it('refuses a symlinked native shadow sidecar before opening the artifact', async () => { + const options = await setup(); + await store.close(); + const target = path.join(options.directory, 'unrelated-file'); + await fs.writeFile(target, 'retain this'); + await fs.symlink(target, path.join(options.directory, 'vectors.shadow')); + await expect(VectorStore.open({ ...options, mode: 'resume' })).rejects.toThrow( + 'owned regular file', + ); + expect(await fs.readFile(target, 'utf8')).toBe('retain this'); + }); + it('never bootstraps over an unidentified nonempty native catalog', async () => { + temporary = await createTempDir(); + const options = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'resume' as const, + recoverEmptyBootstrap: true, + }; + const file = path.join(options.directory, 'vectors'); + let handle = await openLbugConnection(lbug, file, { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + const execute = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + await execute('CREATE NODE TABLE PreviousWork (id STRING, PRIMARY KEY(id))'); + await execute("CREATE (:PreviousWork {id:'retain-this'})"); + await closeLbugConnection(handle); + await expect(VectorStore.open(options)).rejects.toThrow(); + handle = await openLbugConnection(lbug, file, { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query('MATCH (n:PreviousWork) RETURN n.id AS id'); + const parts = Array.isArray(result) ? result : [result]; + expect(parts[0].getNextSync()).toEqual({ id: 'retain-this' }); + parts.forEach((part) => part.close()); + } finally { + await closeLbugConnection(handle); + } + }); + it('recovers an interrupted vector bootstrap only with explicit empty-catalog admission', async () => { + temporary = await createTempDir(); + const options = { + directory: temporary.dbPath, + identity: 'a'.repeat(64), + dimensions: 4, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + mode: 'create' as const, + }; + await expect( + promisify(execFile)(process.execPath, [ + fileURLToPath(new URL('../helpers/interrupted-vector-bootstrap.mjs', import.meta.url)), + JSON.stringify(options), + ]), + ).rejects.toMatchObject({ code: 73 }); + await expect(VectorStore.open({ ...options, mode: 'resume' })).rejects.toThrow(); + store = await VectorStore.open({ ...options, mode: 'resume', recoverEmptyBootstrap: true }); + await store.beginNode('Function:a:test', 'hash'); + await store.append('Function:a:test', 'hash', [chunk(0)]); + const receipt = await store.finishNode('Function:a:test', 'hash', 1); + await store.checkpointUnit([receipt]); + expect(await store.reusable('Function:a:test', 'hash')).toMatchObject({ + complete: true, + chunks: 1, + }); + }); + + it.each(['checksum', 'missing', 'accounting'])( + 'recovers a damaged failed-unit %s without repeating acknowledged provider requests', + async (damage) => { + const fixture = await runnerFixture(65); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00064'))) + throw new HttpEmbeddingError('bad input', { status: 400 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + expect((await fixture.run()).state).toMatchObject({ + completedUnits: 1, + failedUnits: 1, + completedNodes: 64, + }); + const failedFile = path.join(fixture.options.directory, 'failed-0000000000000001.json'); + if (damage === 'missing') await fs.unlink(failedFile); + else if (damage === 'checksum') await fs.writeFile(failedFile, '{broken'); + else { + const { readEmbeddingJobRecord, writeEmbeddingJobRecord } = + await import('../../src/storage/embedding-job.js'); + const record = (await readEmbeddingJobRecord( + fixture.options.directory, + path.basename(failedFile), + )) as { intent: { nodes: number } }; + await writeEmbeddingJobRecord(fixture.options.directory, path.basename(failedFile), { + ...record, + intent: { ...record.intent, nodes: 2 }, + }); + } + await fixture.reopen(true); + fixture.provider.mockClear(); + fixture.provider.mockImplementation(async (texts) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + const result = await fixture.run(); + expect(result.state).toMatchObject({ + phase: 'indexing', + failedUnits: 0, + reusedNodes: 64, + completedNodes: 1, + chunks: 65, + }); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect( + (await fs.readdir(fixture.options.directory)).some((name) => + name.startsWith('job.json.preserved-'), + ), + ).toBe(true); + }, + ); + it('copies compatible complete vectors into a new generation without re-embedding', async () => { + const fixture = await runnerFixture(3, 'const value = "漢😀";'.repeat(300)); + const first = await fixture.run(); + const previous = store; + const directory = path.join(fixture.options.directory, 'next-generation'); + await fs.mkdir(directory); + const manifest = await createEmbeddingWorkManifest(directory, 'new-source', fixture.source); + const identity = 'b'.repeat(64); + const journal = await EmbeddingJobJournal.create(directory, identity, manifest); + store = await VectorStore.open({ ...fixture.options, directory, identity }); + const shiftedSource = { + ...fixture.source, + fetch: async (label: string, ids: readonly string[]) => + (await fixture.source.fetch(label, ids)).map((node) => ({ + ...node, + startLine: 50, + endLine: 50, + })), + }; + const provider = vi.fn(async () => { + throw new Error('Reuse must not call provider'); + }); + try { + const result = await runEmbeddingJob({ + directory, + manifest, + source: shiftedSource, + journal, + store, + reuse: previous, + provider, + config: { maxInputChars: 1024 }, + queueBytes: 32 * 1024 * 1024, + }); + expect(result.state).toMatchObject({ + reusedNodes: 3, + completedNodes: 0, + chunks: first.state.chunks, + }); + expect(provider).not.toHaveBeenCalled(); + const copied = await store.completedNodes(['Const:source.ts:n00000']); + const receipt = copied.get('Const:source.ts:n00000'); + expect(receipt).toMatchObject({ sourceStartLine: 50, sourceEndLine: 50 }); + expect((await store.readChunks(receipt, 0, 1))[0]).toMatchObject({ + startLine: 50, + endLine: 50, + }); + } finally { + await previous.close(); + } + }); + + it.each(['before', 'after'] as const)( + 'replaces an interrupted chunk group after a %s-write failure', + async (when) => { + const fixture = await runnerFixture(65); + const append = store.appendBatch.bind(store); + vi.spyOn(store, 'appendBatch').mockImplementation(async (rows) => { + if (rows.some((row) => row.nodeId === 'Const:source.ts:n00064')) { + if (when === 'after') await append(rows); + throw new Error('simulated write failure'); + } + await append(rows); + }); + await expect(fixture.run()).rejects.toThrow('write failure'); + expect(fixture.journal.snapshot()).toMatchObject({ + completedNodes: 64, + active: { start: { unit: 1 } }, + }); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).state).toMatchObject({ + completedNodes: 65, + chunks: 65, + failedUnits: 0, + }); + const paid = fixture.provider.mock.calls.flatMap(([texts]) => texts); + expect(paid).toHaveLength(1); + expect(paid[0]).toContain('n00064'); + }, + ); + + it('preserves a corrupt journal and reconstructs complete native coverage without provider calls', async () => { + const fixture = await runnerFixture(65); + await fixture.run(); + const recordPath = path.join(fixture.options.directory, 'job.json'); + const damaged = '{"checksum":"bad","payload":{}}'; + await fs.writeFile(recordPath, damaged); + await fixture.reopen(true); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + const result = await fixture.run(); + expect(result.state).toMatchObject({ + reusedNodes: 65, + chunks: 65, + failedUnits: 0, + phase: 'indexing', + }); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(fixture.source.fetch).not.toHaveBeenCalled(); + const saved = (await fs.readdir(fixture.options.directory)).filter((name) => + name.startsWith('job.json.preserved-'), + ); + expect(saved).toHaveLength(1); + expect(await fs.readFile(path.join(fixture.options.directory, saved[0]), 'utf8')).toBe(damaged); + }); + + it('scans only keys and byte bounds, then fetches a bounded PK unit from an immutable graph', async () => { + const options = await setup(); + const graphPath = path.join(options.directory, 'source-graph'); + const handle = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + try { + await query( + 'CREATE NODE TABLE File(id STRING, name STRING, filePath STRING, content STRING, PRIMARY KEY(id))', + ); + await query( + 'CREATE NODE TABLE Function(id STRING, name STRING, filePath STRING, content STRING, description STRING, startLine INT64, endLine INT64, isExported BOOL, PRIMARY KEY(id))', + ); + await query( + "CREATE (:Function {id:'Function:a:漢', name:'漢', filePath:'a.ts', content:'漢😀\ntext', startLine:2, endLine:3, isExported:true})", + ); + await query("CREATE (:File {id:'File:a', name:'a', filePath:'a.md', content:'漢😀\ntext'})"); + await query( + "CREATE (:File {id:'File:blank', name:'blank', filePath:'blank.md', content:'\u00a0\u3000\ufeff'})", + ); + await query( + "CREATE (:File {id:'File:binary', name:'binary', filePath:'binary.dat', content:'[Binary file - content not stored]'})", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle); + } + const source = await EmbeddingGraphSource.open(graphPath, 64 * 1024 * 1024, 1); + try { + const keys = []; + for await (const key of source.scan('Function')) keys.push(key); + expect(keys).toEqual([ + { id: 'Function:a:漢', contentBytes: 12, metadataBytes: expect.any(Number) }, + ]); + const nodes = await source.fetch('Function', [keys[0].id]); + expect(nodes[0]).toMatchObject({ + id: 'Function:a:漢', + content: '漢😀\ntext', + startLine: 2, + endLine: 3, + isExported: true, + }); + expect(keys[0].metadataBytes).toBeGreaterThanOrEqual( + Buffer.byteLength(JSON.stringify({ ...nodes[0], content: undefined })), + ); + const files = []; + for await (const key of source.scan('File')) files.push(key.id); + expect(files).toEqual(['File:a']); + expect((await source.fetch('File', files))[0]).toMatchObject({ startLine: 1, endLine: 2 }); + await expect(source.fetch('Function', Array(65).fill('id'))).rejects.toThrow('unit'); + await expect(source.scan('Unknown').next()).rejects.toThrow('label'); + } finally { + await source.close(); + } + }); + + it('recovers native durability before cursor acknowledgement without repeating paid inputs', async () => { + const fixture = await runnerFixture(130); + const acknowledge = fixture.journal.acknowledge.bind(fixture.journal); + let acks = 0; + vi.spyOn(fixture.journal, 'acknowledge').mockImplementation(async (result) => { + if (++acks === 2) throw new Error('simulated cursor write failure'); + await acknowledge(result); + }); + await expect(fixture.run()).rejects.toThrow('cursor write'); + expect(fixture.journal.snapshot()).toMatchObject({ + completedNodes: 64, + active: { start: { unit: 1 } }, + }); + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + const result = await fixture.run({ + queueBytes: 8 * 1024 * 1024, + config: { subBatchSize: 2, maxInputChars: 1024 }, + }); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(2); + expect(fixture.source.fetch.mock.calls.flatMap(([, ids]) => ids)).toHaveLength(2); + expect(result.state).toMatchObject({ + phase: 'indexing', + completedNodes: 66, + reusedNodes: 64, + failedUnits: 0, + chunks: 130, + }); + // A crash after seal but before publication requires no model/source work. + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.source.fetch.mockClear(); + expect((await fixture.run()).indexKind).toBe('exact'); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(fixture.source.fetch).not.toHaveBeenCalled(); + }); + + it('keeps one rejected input retryable while preserving every healthy sibling', async () => { + const fixture = await runnerFixture(12); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00003'))) + throw new HttpEmbeddingError('bad input', { status: 400 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + const first = await fixture.run(); + expect(first.indexKind).toBeUndefined(); + expect(first.state).toMatchObject({ failedUnits: 1, phase: 'embedding', active: null }); + expect( + ( + await store.completedNodes( + Array.from({ length: 12 }, (_, i) => `Const:source.ts:n${String(i).padStart(5, '0')}`), + ) + ).size, + ).toBe(11); + await fixture.reopen(); + fixture.provider.mockClear(); + fixture.provider.mockImplementation(async (texts) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])), + ); + fixture.source.fetch.mockClear(); + const result = await fixture.run(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(fixture.source.fetch.mock.calls[0][1]).toEqual(['Const:source.ts:n00003']); + expect(result.state).toMatchObject({ + failedUnits: 0, + completedNodes: 1, + reusedNodes: 11, + chunks: 12, + }); + }); + + it('excludes a permanently rejected node after three invocations and seals its healthy siblings', async () => { + const fixture = await runnerFixture(12); + fixture.provider.mockImplementation(async (texts) => { + if (texts.some((text) => text.includes('n00003'))) + throw new HttpEmbeddingError('bad input', { status: 422 }); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + await fixture.run(); + await fixture.reopen(); + fixture.provider.mockClear(); + expect((await fixture.run()).indexKind).toBeUndefined(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + await fixture.reopen(); + fixture.provider.mockClear(); + const final = await fixture.run(); + expect(fixture.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(1); + expect(final.state).toMatchObject({ + failedUnits: 0, + excludedNodes: 1, + reusedNodes: 11, + completedNodes: 0, + chunks: 11, + phase: 'indexing', + }); + expect(final.indexKind).toBe('exact'); + expect(store.isSealed).toBe(true); + fixture.provider.mockClear(); + await fixture.run(); + expect(fixture.provider).not.toHaveBeenCalled(); + }); + + it('bounds one oversized node by bytes and preserves all generated Unicode input', async () => { + const content = '漢😀abcdef'.repeat(60000); + const fixture = await runnerFixture(1, content); + let chars = 0; + let requests = 0; + // Do not retain the fixture's requests: test the actual production queue. + const provider = async (texts: string[]) => { + requests++; + expect(texts.length).toBeLessThanOrEqual(8); + expect(texts.every((text) => text.length <= 1024)).toBe(true); + expect(texts.every((text) => !/[\uD800-\uDBFF]$/.test(text))).toBe(true); + chars += texts.reduce((n, text) => n + text.length, 0); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }; + const result = await fixture.run({ provider }); + expect(chars).toBeGreaterThan(content.length); + expect(chars).toBeLessThan(content.length + 100); + expect(requests).toBeGreaterThan(50); + expect(result.peakQueueBytes).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(result.state.completedNodes).toBe(1); + }); + + it('rejects impossible admission before intent, provider initialization, or vector mutation', async () => { + const fixture = await runnerFixture(1, 'x'.repeat(100000)); + const begin = vi.spyOn(store, 'beginNodes'); + await expect(fixture.run({ queueBytes: 1024 * 1024 })).rejects.toThrow( + 'larger resource budget', + ); + expect(fixture.initializeProvider).not.toHaveBeenCalled(); + expect(fixture.provider).not.toHaveBeenCalled(); + expect(begin).not.toHaveBeenCalled(); + expect(fixture.journal.snapshot().active).toBeNull(); + }); + + it('does not bisect endpoint outages and stops at the existing consecutive-failure ceiling', async () => { + const fixture = await runnerFixture(100); + fixture.provider.mockRejectedValue(new HttpEmbeddingError('unavailable', { status: 503 })); + await expect(fixture.run()).rejects.toThrow('unavailable'); + expect(fixture.provider).toHaveBeenCalledTimes(5); + expect(fixture.journal.snapshot()).toMatchObject({ + failedUnits: 1, + completedNodes: 0, + active: null, + }); + }); + + it('leaves a durable active unit on cancellation and never writes a late provider response', async () => { + const fixture = await runnerFixture(20); + const controller = new AbortController(); + const append = vi.spyOn(store, 'appendBatch'); + fixture.provider.mockImplementation(async (texts) => { + controller.abort(); + return texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])); + }); + await expect(fixture.run({ signal: controller.signal })).rejects.toThrow(); + expect(fixture.provider).toHaveBeenCalledTimes(1); + expect(append).not.toHaveBeenCalled(); + expect(fixture.journal.snapshot().active).not.toBeNull(); + }); +}); + +describe('separate durable vector artifacts', () => { + it('publishes exact-input digests with rows and updates a verified source hash without replacing vectors', async () => { + const options = await setup(); + const nodeId = 'Function:source.ts:sample', + hash = 'a'.repeat(40), + digest = 'b'.repeat(40); + await store.beginNodes([{ nodeId, contentHash: hash, sourceStartLine: 1, sourceEndLine: 2 }]); + await store.appendBatch([{ nodeId, contentHash: hash, ...chunk(0) }]); + const [receipt] = await store.finishNodes([ + { nodeId, contentHash: hash, chunks: 1, inputsDigest: digest }, + ]); + expect(receipt.contentHash).toBe(`${hash}:${digest}`); + const original = await store.readChunks(receipt, 0, 1); + const next = { + nodeId, + contentHash: `${'c'.repeat(40)}:${digest}`, + sourceStartLine: 1, + sourceEndLine: 2, + }; + expect(await store.reuseNode(next, async () => undefined)).toBeNull(); + const reused = await store.reuseNode(next, async (previous) => + previous === receipt.contentHash ? next.contentHash : undefined, + ); + expect(reused?.contentHash).toBe(next.contentHash); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + const recovered = (await store.completedNodes([nodeId])).get(nodeId); + if (!recovered) throw new Error('Missing completed input receipt'); + expect(recovered.contentHash).toBe(next.contentHash); + expect(await store.readChunks(recovered, 0, 1)).toEqual(original); + }); + it('preserves each node, chunk ordinal, line range and vector in a mixed batch', async () => { + const options = await setup(); + const nodes = [0, 1].map((i) => ({ + nodeId: `Function:a:mixed${i}`, + contentHash: `hash${i}`, + sourceStartLine: i * 10, + sourceEndLine: i * 10 + 5, + })); + const rows = [ + { ...nodes[0], chunkIndex: 0, startLine: 0, endLine: 2, embedding: [1, 2, 3, 4] }, + { ...nodes[1], chunkIndex: 0, startLine: 10, endLine: 15, embedding: [5, 6, 7, 8] }, + { ...nodes[0], chunkIndex: 1, startLine: 3, endLine: 5, embedding: [9, 10, 11, 12] }, + ]; + await store.beginNodes(nodes); + await store.appendBatch(rows); + const receipts = await store.finishNodes( + nodes.map((node, i) => ({ ...node, chunks: i === 0 ? 2 : 1 })), + ); + await store.checkpointUnit(receipts); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + for (const receipt of receipts) { + expect(await store.readChunks(receipt, 0, receipt.chunks)).toEqual( + rows + .filter((row) => row.nodeId === receipt.nodeId) + .map(({ chunkIndex, startLine, endLine, embedding }) => ({ + chunkIndex, + startLine, + endLine, + embedding, + })), + ); + } + }); + + it('verifies a complete unit with bounded native queries instead of one query per node', async () => { + await setup(); + const nodes = Array.from({ length: 64 }, (_, i) => ({ + nodeId: `Function:a:batch${i}`, + contentHash: `hash${i}`, + })); + await store.beginNodes(nodes); + await store.appendBatch(nodes.map((node) => ({ ...node, ...chunk(0) }))); + const execute = vi.spyOn(lbug.Connection.prototype, 'execute'); + const receipts = await store.finishNodes(nodes.map((node) => ({ ...node, chunks: 1 }))); + expect(receipts).toHaveLength(64); + expect(receipts.every((receipt) => receipt.complete)).toBe(true); + expect(execute.mock.calls.length).toBeLessThanOrEqual(3); + execute.mockRestore(); + await store.checkpointUnit(receipts); + }); + + it('rejects a cross-node chunk mismatch before completing any group in a unit', async () => { + const options = await setup(); + const nodes = [0, 1].map((i) => ({ nodeId: `Function:a:group${i}`, contentHash: `hash${i}` })); + await store.beginNodes(nodes); + await store.appendBatch(nodes.map((node) => ({ ...node, ...chunk(0) }))); + await store.close(); + store = undefined; + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query( + "MATCH (e:CodeEmbedding {id:'Function:a:group1:0'}) SET e.nodeId='Function:a:group0'", + ); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } finally { + await closeLbugConnection(handle); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await expect(store.finishNodes(nodes.map((node) => ({ ...node, chunks: 1 })))).rejects.toThrow( + 'invalid chunk set', + ); + for (const node of nodes) + expect(await store.reusable(node.nodeId, node.contentHash)).toBeNull(); + }); + + it('finalizes distinct input hashes across chunk batches and preserves unhashed groups on reopen', async () => { + const options = await setup(); + const nodes = [0, 1, 2].map((i) => ({ + nodeId: `Function:a:node${i}`, + contentHash: String(i).repeat(40), + })); + await store.beginNodes(nodes); + for (let start = 0; start < 601; start += 128) { + const rows = Array.from({ length: Math.min(128, 601 - start) }, (_, i) => ({ + ...nodes[Math.floor((start + i) / 300)], + ...chunk((start + i) % 300), + })); + await store.appendBatch(rows); + } + const receipts = await store.finishNodes( + nodes.map((node, i) => ({ + ...node, + chunks: i < 2 ? 300 : 1, + inputsDigest: i < 2 ? String(i + 3).repeat(40) : undefined, + })), + ); + expect(receipts.map((receipt) => receipt.contentHash)).toEqual([ + `${nodes[0].contentHash}:${'3'.repeat(40)}`, + `${nodes[1].contentHash}:${'4'.repeat(40)}`, + nodes[2].contentHash, + ]); + await store.checkpointUnit(receipts); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + for (const receipt of receipts) + expect(await store.reusable(receipt.nodeId, receipt.contentHash)).toEqual(receipt); + expect(await store.seal(3, 601)).toBe('exact'); + }); + + it('publishes an empty artifact without loading a model or ANN extension', async () => { + const options = await setup(); + expect(await store.seal(0, 0)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + }); + + it('replays incomplete groups and removes surplus old chunks without damaging completed groups', async () => { + const options = await setup(); + await store.beginNode('Function:a:complete', 'hash'); + await store.append('Function:a:complete', 'hash', [chunk(0)]); + const complete = await store.finishNode('Function:a:complete', 'hash', 1); + await store.checkpointUnit([complete]); + await store.beginNode('Function:a:active', 'old'); + await store.append('Function:a:active', 'old', [chunk(0), chunk(1), chunk(2)]); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'resume' }); + expect(await store.reusable('Function:a:active', 'old')).toBeNull(); + expect(await store.reusable('Function:a:complete', 'hash')).toEqual(complete); + await store.beginNode('Function:a:active', 'new'); + await store.append('Function:a:active', 'new', [chunk(0)]); + const active = await store.finishNode('Function:a:active', 'new', 1); + await store.checkpointUnit([active]); + expect(await store.seal(2, 2)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.reusable('Function:a:active', 'new')).toEqual(active); + await expect(store.beginNode('Function:a:active', 'other')).rejects.toThrow('immutable'); + }); + + it('rejects invalid batches before mutation and never acknowledges a partial node', async () => { + await setup(); + await store.beginNode('Function:a:node', 'hash'); + for (const invalid of [ + chunk(1), + { ...chunk(0), embedding: [NaN, 0, 0, 0] }, + { ...chunk(0), embedding: [0, 0] }, + { ...chunk(0), endLine: 0 }, + ]) { + await expect(store.append('Function:a:node', 'hash', [invalid])).rejects.toThrow(); + } + await store.append('Function:a:node', 'hash', [chunk(0)]); + await expect( + store.checkpointUnit([ + { nodeId: 'Function:a:node', contentHash: 'hash', chunks: 1, complete: true }, + ]), + ).rejects.toThrow('incomplete'); + await expect(store.finishNode('Function:a:node', 'hash', 2)).rejects.toThrow('coverage'); + await expect(store.seal(1, 1)).rejects.toThrow('incomplete'); + const receipt = await store.finishNode('Function:a:node', 'hash', 1); + await store.checkpointUnit([receipt]); + }); + + it('refuses identity mismatch, missing chunks, and unpublished readers', async () => { + const options = await setup(); + await store.beginNode('Function:a:node', 'hash'); + await store.append('Function:a:node', 'hash', [chunk(0), chunk(1)]); + const receipt = await store.finishNode('Function:a:node', 'hash', 2); + await store.checkpointUnit([receipt]); + await store.close(); + store = undefined; + await expect(VectorStore.open({ ...options, mode: 'read' })).rejects.toThrow('not complete'); + await expect( + VectorStore.open({ ...options, mode: 'resume', identity: 'b'.repeat(64) }), + ).rejects.toThrow('identity'); + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const result = await handle.conn.query( + "MATCH (e:CodeEmbedding {id:'Function:a:node:1'}) DELETE e", + ); + for (const part of Array.isArray(result) ? result : [result]) part.close(); + } finally { + await closeLbugConnection(handle); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await expect(store.reusable('Function:a:node', 'hash')).rejects.toThrow('missing chunks'); + }); + + it('keeps a separately published graph readable while the vector writer is open', async () => { + const options = await setup(); + const graphPath = path.join(options.directory, 'graph'); + let graph = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const run = async (sql: string) => { + const result = await graph.conn.query(sql); + const parts = Array.isArray(result) ? result : [result]; + try { + return await parts[0].getAll(); + } finally { + parts.forEach((part) => part.close()); + } + }; + await run('CREATE NODE TABLE Function(id STRING, PRIMARY KEY(id))'); + await run("CREATE (:Function {id:'Function:a:node'})"); + await run('CHECKPOINT'); + await closeLbugConnection(graph); + graph = await openLbugConnection(lbug, graphPath, { + readOnly: true, + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + await store.beginNode('Function:a:node', 'hash'); + await store.append('Function:a:node', 'hash', [chunk(0)]); + expect(await run('MATCH (n:Function) RETURN n.id AS id')).toEqual([ + { id: 'Function:a:node' }, + ]); + } finally { + await closeLbugConnection(graph); + } + }); + + it('does not replace an existing artifact or accept an unbounded reservation', async () => { + const options = await setup(); + await expect(VectorStore.open(options)).rejects.toThrow('exists'); + await expect(VectorStore.open({ ...options, bufferPoolBytes: 0 })).rejects.toThrow( + 'reservation', + ); + expect((await fs.stat(path.join(options.directory, 'vectors'))).isFile()).toBe(true); + }); +}); + +async function loadVectorFixture(directory: string): Promise { + const probe = await openLbugConnection(lbug, path.join(directory, 'extension-probe'), { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + return await new ExtensionManager().ensure( + async (sql) => { + const result = await probe.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) { + try { + await cursor.getAll(); + } finally { + cursor.close(); + } + } + }, + 'vector', + 'vector fixture', + { policy: 'load-only' }, + ); + } finally { + await closeLbugConnection(probe); + } +} + +describe('published vector readers', () => { + it('seals a 20001-row 384-dimensional artifact within the admitted writer pool', async (context) => { + const initial = await setup(); + await store.close(); + await fs.rm(path.join(initial.directory, 'vectors'), { force: true }); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + const loaded = await loadVectorFixture(initial.directory); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } + expect(loaded).toBe(true); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + store = await VectorStore.open({ + ...initial, + dimensions: 384, + batchBytes: 2 * 1024 ** 2, + // The controller reserves the other half for a compatible reuse reader. + bufferPoolBytes: Math.floor(resources.embedding.vectorPoolBytes / 2), + }); + const nodeId = 'Function:source.ts:large'; + await store.beginNode(nodeId, 'hash'); + const total = 20001; + for (let start = 0; start < total; start += 256) { + await store.append( + nodeId, + 'hash', + Array.from({ length: Math.min(256, total - start) }, (_, offset) => ({ + chunkIndex: start + offset, + startLine: start + offset + 1, + endLine: start + offset + 1, + embedding: Array.from({ length: 384 }, (_, dimension) => + dimension === 0 ? 1 : ((start + offset + dimension) % 997) / 997, + ), + })), + ); + } + await store.finishNode(nodeId, 'hash', total); + const sealed = await store.seal(1, total); + expect(sealed).toBe('hnsw'); + expect(store.isSealed).toBe(true); + const options = store.options; + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.seal(1, total)).toBe('hnsw'); + }, 120_000); + + it('resumes writes after a crash between HNSW creation and publication', async (context) => { + const options = await setup(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + const nodeId = 'Function:source.ts:alpha'; + await store.beginNode(nodeId, 'old'); + await store.append(nodeId, 'old', [chunk(0)]); + await store.finishNode(nodeId, 'old', 1); + await store.checkpointUnit( + [...(await store.completedNodes([nodeId]))].map(([, receipt]) => receipt), + ); + await store.close(); + const handle = await openLbugConnection(lbug, path.join(options.directory, 'vectors'), { + bufferPoolBytes: options.bufferPoolBytes, + threads: 1, + }); + try { + const query = async (sql: string) => { + const result = await handle.conn.query(sql); + for (const cursor of Array.isArray(result) ? result : [result]) { + try { + await cursor.getAll(); + } finally { + cursor.close(); + } + } + }; + const loaded = await new ExtensionManager().ensure( + query, + 'vector', + 'interrupted vector fixture', + { policy: 'load-only' }, + ); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } + expect(loaded).toBe(true); + await query( + "CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')", + ); + await query('CHECKPOINT'); + } finally { + await closeLbugConnection(handle, { + strict: true, + databasePath: path.join(options.directory, 'vectors'), + }); + } + store = await VectorStore.open({ ...options, mode: 'resume' }); + await store.beginNode(nodeId, 'new'); + await store.append(nodeId, 'new', [chunk(0), chunk(1)]); + await store.finishNode(nodeId, 'new', 2); + expect(await store.seal(1, 2)).toBe('hnsw'); + }); + + it('shares one native database across independently pinned vector files', async () => { + const options = await setup(); + const id = 'Function:source.ts:alpha'; + await store.beginNode(id, 'hash'); + await store.append(id, 'hash', [chunk(0)]); + await store.finishNode(id, 'hash', 1); + await store.seal(1, 1); + await store.close(); + const first = path.join(options.directory, 'vectors'); + const second = path.join(options.directory, 'sibling-vectors'); + await fs.link(first, second); + const close = vi.spyOn(lbug.Database.prototype, 'close'); + const reservation = { kind: 'vector' as const, bufferPoolBytes: 64 * 1024 * 1024, threads: 1 }; + try { + await readerPool.initLbug('vector-first', first, reservation); + await readerPool.initLbug('vector-second', second, reservation); + await readerPool.closeLbug('vector-first'); + expect(close).not.toHaveBeenCalled(); + // POSIX can unlink the retired path while its immutable inode is open. + // Windows may retain that path until the last reader releases it. + if (process.platform !== 'win32') await fs.unlink(first); + expect( + await readerPool.executeQuery( + 'vector-second', + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS id', + ), + ).toEqual([{ id }]); + await readerPool.closeLbug('vector-second'); + expect(close).toHaveBeenCalledOnce(); + await readerPool.initLbug('vector-second', second, reservation); + expect( + await readerPool.executeQuery( + 'vector-second', + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS id', + ), + ).toEqual([{ id }]); + } finally { + await readerPool.closeLbug(); + close.mockRestore(); + } + }); + + it('publishes a native FTS graph before opening the independent vector writer', async (context) => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(64 * 1024 * 1024)); + const root = await fs.realpath(temporary.dbPath); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + const graph = await openLbugConnection(lbug, stage, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + const query = async (sql: string) => { + const result = await graph.conn.query(sql); + const rows = []; + for (const part of Array.isArray(result) ? result : [result]) { + try { + rows.push(...(await part.getAll())); + } finally { + await part.close(); + } + } + return rows; + }; + let prepared; + try { + const ready = await new ExtensionManager().ensure(query, 'fts', 'publication fixture', { + policy: 'load-only', + }); + if (!ready) { + if (process.env.GITNEXUS_REQUIRE_FTS === '1') throw new Error('Native FTS is required'); + context.skip('Optional native FTS extension is unavailable'); + return; + } + await query('CREATE NODE TABLE Function (id STRING, name STRING, PRIMARY KEY(id))'); + await query("CREATE (:Function {id:'Function:source.ts:alpha', name:'alpha'})"); + await query("CALL CREATE_FTS_INDEX('Function', 'function_fts', ['name'])"); + await query('CHECKPOINT'); + prepared = await prepareAnalysisPublication( + root, + stage, + { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + stats: { nodes: 1 }, + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }, + 'c'.repeat(64), + ); + } finally { + await closeLbugConnection(graph); + } + const metadata = await publishPreparedAnalysis(root, { + verify: async (file) => { + await readerPool.initLbug('verify-publication', file); + try { + expect( + await readerPool.executeQuery( + 'verify-publication', + "CALL QUERY_FTS_INDEX('Function', 'function_fts', 'alpha') YIELD node, score RETURN node.name AS name", + ), + ).toEqual([{ name: 'alpha' }]); + } finally { + await readerPool.closeLbug('verify-publication'); + } + }, + saveMetadata: (meta) => saveMeta(root, meta), + }); + expect(metadata?.graphPath).toBe(prepared.graphFile); + const publishedPath = path.join(root, prepared.graphFile); + const directory = path.join(root, 'embeddings', 'd'.repeat(64)); + await fs.mkdir(directory, { recursive: true }); + try { + await readerPool.initLbug('warm-fts', publishedPath); + store = await VectorStore.open({ + directory, + identity: 'd'.repeat(64), + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0)]); + await readerPool.initLbug('cold-fts', publishedPath); + for (const key of ['warm-fts', 'cold-fts']) { + expect( + await readerPool.executeQuery( + key, + "CALL QUERY_FTS_INDEX('Function', 'function_fts', 'alpha') YIELD node, score RETURN node.name AS name", + ), + ).toEqual([{ name: 'alpha' }]); + } + } finally { + await readerPool.closeLbug(); + } + }); + + it('uses the native ANN lane when exact scan cannot cover the artifact', async (context) => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '1'); + const root = temporary.dbPath; + const loaded = await loadVectorFixture(root); + if (!loaded && process.env.GITNEXUS_REQUIRE_VECTOR !== '1') { + context.skip('Optional native VECTOR extension is unavailable'); + return; + } + expect(loaded).toBe(true); + const generation = 'a'.repeat(32); + const identity = 'b'.repeat(64); + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + const options: VectorStoreOptions = { + directory, + identity, + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }; + store = await VectorStore.open(options); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + await store.finishNode('Function:source.ts:alpha', 'hash', 2); + const kind = await store.seal(1, 2); + expect(kind).toBe('hnsw'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect(await store.seal(1, 2)).toBe('hnsw'); + await store.close(); + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: `lbug.generation.${generation}`, + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: createHash('sha256') + .update(JSON.stringify(['local', 'fixture', 4])) + .digest('hex'), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind: kind, + }, + }; + try { + const found = await searchVectorGeneration( + { metadataDirectory: root, metadata, graphPath: path.join(root, metadata.graphPath!) }, + async () => [0.1, 0.2, 0.3, 0.4], + 5, + 1, + async () => ({ provider: 'local', model: 'fixture', dimensions: 4 }), + ); + expect(found).toEqual([ + expect.objectContaining({ nodeId: 'Function:source.ts:alpha', startLine: 1, endLine: 2 }), + ]); + const execute = readerPool.executeParameterized; + vi.spyOn(readerPool, 'executeParameterized').mockImplementation((db, sql, params) => + sql.includes('QUERY_VECTOR_INDEX') ? Promise.resolve([]) : execute(db, sql, params), + ); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '10'); + expect( + await searchVectorGeneration( + { metadataDirectory: root, metadata, graphPath: path.join(root, metadata.graphPath!) }, + async () => [0.1, 0.2, 0.3, 0.4], + 5, + 1, + async () => ({ provider: 'local', model: 'fixture', dimensions: 4 }), + ), + ).toEqual([ + expect.objectContaining({ + nodeId: 'Function:source.ts:alpha', + chunkIndex: 0, + startLine: 1, + endLine: 2, + distance: expect.any(Number), + }), + ]); + } finally { + await readerPool.closeLbug(); + } + }); + + it('serves warm and cold graph readers while vectors are unpublished, then searches the closed artifact concurrently', async () => { + temporary = await createTempDir(); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(64 * 1024 * 1024)); + const root = temporary.dbPath; + const generation = 'a'.repeat(32); + const identity = 'b'.repeat(64); + const graphPath = path.join(root, `lbug.generation.${generation}`); + const graph = await openLbugConnection(lbug, graphPath, { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + }); + try { + for (const query of [ + 'CREATE NODE TABLE Function (id STRING, name STRING, PRIMARY KEY(id))', + "CREATE (:Function {id:'Function:source.ts:alpha', name:'alpha'})", + 'CHECKPOINT', + ]) { + const result = await graph.conn.query(query); + for (const part of Array.isArray(result) ? result : [result]) { + await part.getAll(); + await part.close(); + } + } + } finally { + await closeLbugConnection(graph); + } + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + store = await VectorStore.open({ + directory, + identity, + dimensions: 4, + mode: 'create', + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 64 * 1024, + }); + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: path.basename(graphPath), + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + embeddingJob: { version: 1, identity, graphGeneration: generation }, + }; + const snapshot = { metadataDirectory: root, metadata, graphPath }; + await saveMeta(root, metadata); + const embed = vi.fn(async () => [0.1, 0.2, 0.3, 0.4]); + const vectorIdentity = { provider: 'local', model: 'fixture', dimensions: 4 }; + const resolveIdentity = async () => vectorIdentity; + try { + await readerPool.initLbug(graphPath, graphPath); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + expect(await searchVectorGeneration(snapshot, embed, 5, 1, resolveIdentity)).toEqual([]); + expect(embed).not.toHaveBeenCalled(); + await readerPool.initLbug('cold-reader', graphPath); + for (const key of [graphPath, 'cold-reader']) { + expect( + await readerPool.executeQuery(key, 'MATCH (n:Function) RETURN n.name AS name'), + ).toEqual([{ name: 'alpha' }]); + } + const receipt = await store.finishNode('Function:source.ts:alpha', 'hash', 2); + await store.checkpointUnit([receipt]); + const indexKind = await store.seal(1, 2); + await store.close(); + const publishedGeneration: NonNullable = { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: createHash('sha256') + .update(JSON.stringify(['local', 'fixture', 4])) + .digest('hex'), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind, + }; + const resources = { bufferPoolBytes: 64 * 1024 * 1024, threads: 1, batchBytes: 64 * 1024 }; + const actions = { + loadMetadata: () => loadMeta(root), + saveMetadata: (meta: RepoMeta) => saveMeta(root, meta), + }; + await expect( + publishVectorGeneration(root, { ...publishedGeneration, nodes: 2 }, resources, actions), + ).rejects.toThrow('incomplete'); + expect((await loadMeta(root))?.vectorGeneration).toBeUndefined(); + const published = await publishVectorGeneration( + root, + publishedGeneration, + resources, + actions, + ); + expect(published?.embeddingJob).toBeUndefined(); + expect(published?.stats?.embeddings).toBe(2); + snapshot.metadata = published!; + const results = await Promise.all( + Array.from({ length: 8 }, () => + searchVectorGeneration(snapshot, embed, 5, 1, resolveIdentity), + ), + ); + for (const result of results) + expect(result).toEqual([ + expect.objectContaining({ + nodeId: 'Function:source.ts:alpha', + chunkIndex: 0, + startLine: 1, + endLine: 2, + }), + ]); + expect(embed).toHaveBeenCalledTimes(8); + embed.mockClear(); + expect( + await searchVectorGeneration(snapshot, embed, 5, 1, async () => ({ + ...vectorIdentity, + model: 'different', + })), + ).toEqual([]); + expect(embed).not.toHaveBeenCalled(); + } finally { + await readerPool.closeLbug(); + } + }); + + it('keeps complete vectors retryable when exact coverage exceeds the reader limit', async () => { + const options = await setup(); + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '1'); + await store.beginNode('Function:source.ts:alpha', 'hash'); + await store.append('Function:source.ts:alpha', 'hash', [chunk(0), chunk(1)]); + await store.finishNode('Function:source.ts:alpha', 'hash', 2); + expect(await store.seal(1, 2)).toBe('exact'); + await store.close(); + await expect(VectorStore.open({ ...options, mode: 'read' })).rejects.toThrow('not complete'); + store = await VectorStore.open({ ...options, mode: 'resume' }); + expect((await store.completedNodes(['Function:source.ts:alpha'])).size).toBe(1); + // Restoring a viable exact policy exercises index-only completion; no + // replacement or provider call is needed to publish the existing rows. + vi.stubEnv('GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT', '2'); + expect(await store.seal(1, 2)).toBe('exact'); + await store.close(); + store = await VectorStore.open({ ...options, mode: 'read' }); + expect((await store.completedNodes(['Function:source.ts:alpha'])).size).toBe(1); + }); +}); diff --git a/gitnexus/test/integration/lbug-readonly-init.test.ts b/gitnexus/test/integration/lbug-readonly-init.test.ts index 1d79ac39c..6682d9d3e 100644 --- a/gitnexus/test/integration/lbug-readonly-init.test.ts +++ b/gitnexus/test/integration/lbug-readonly-init.test.ts @@ -6,11 +6,40 @@ * filesystems such as Docker :ro bind mounts. */ import fs from 'fs/promises'; +import lbug from '@ladybugdb/core'; import { it, expect } from 'vitest'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; import { _initLockPathForTest } from '../../src/core/lbug/lbug-adapter.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; withTestLbugDB('lbug-readonly-init', (handle) => { + it('closes a read-only graph without leaving a checkpoint marker that blocks the next reader', async () => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.closeLbug(); + await adapter.withLbugDb(handle.dbPath, () => adapter.executeQuery('RETURN 1 AS value'), { + readOnly: true, + }); + expect(await adapter.tryFlushWAL()).toBe(false); + await adapter.closeLbug(); + // Open through the raw bounded reader, so the adapter cannot hide a bad + // read-only close by recovering it through a subsequent writable open. + const next = await openLbugConnection(lbug, handle.dbPath, { + readOnly: true, + bufferPoolBytes: 64 * 1024 ** 2, + threads: 1, + }); + try { + const result = await next.conn.query('RETURN 1 AS value'); + const cursors = Array.isArray(result) ? result : [result]; + try { + expect(await cursors[0].getAll()).toEqual([{ value: 1 }]); + } finally { + for (const cursor of cursors) cursor.close(); + } + } finally { + await closeLbugConnection(next, { strict: true, databasePath: handle.dbPath }); + } + }); it('read-only open never creates lbug.init.lock on disk', async () => { const { dbPath } = handle; const lockPath = _initLockPathForTest(dbPath); diff --git a/gitnexus/test/integration/legacy-embedding-controller.test.ts b/gitnexus/test/integration/legacy-embedding-controller.test.ts new file mode 100644 index 000000000..ebc0c0040 --- /dev/null +++ b/gitnexus/test/integration/legacy-embedding-controller.test.ts @@ -0,0 +1,325 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + runPublishedEmbeddingJob, + type PublishedEmbeddingJobOptions, +} from '../../src/core/embeddings/embedding-job.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { LegacyVectorSource } from '../../src/core/lbug/legacy-vector-source.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { recordLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { readEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; + +const roots: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); +async function fixture(count = 130, unsafe: string[] = [], knownProvider = true) { + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-controller-')); + roots.push(root); + const identity = { provider: 'local', model: 'test', dimensions: 4 }; + const config = resolveEmbeddingConfig({ threads: 1, subBatchSize: 8 }); + const sourceFile = 'lbug.generation.' + 'a'.repeat(32); + const sourcePath = path.join(root, sourceFile); + const id = (i: number) => `Const:a.ts:n${String(i).padStart(5, '0')}`; + const node = (key: string): EmbeddableNode => ({ + id: key, + name: key.split(':').at(-1) ?? key, + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, + }); + const handle = await openLbugConnection(lbug, sourcePath, { + bufferPoolBytes: 16 * 1024 * 1024, + threads: 1, + }); + const close = (result: lbug.QueryResult | lbug.QueryResult[]) => { + for (const r of Array.isArray(result) ? result : [result]) r.close(); + }; + try { + close( + await handle.conn.query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ), + ); + const insert = await handle.conn.prepare( + 'CREATE (:CodeEmbedding {id:$id,nodeId:$nodeId,chunkIndex:0,startLine:1,endLine:1,embedding:$embedding,contentHash:$hash})', + ); + for (let i = 0; i < count; i++) + close( + await handle.conn.execute(insert, { + id: id(i) + ':0', + nodeId: id(i), + embedding: [1, 0, 0, 0], + hash: contentHashForNode(node(id(i)), config), + }), + ); + close(await handle.conn.query('CHECKPOINT')); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: sourcePath }); + } + const legacyIdentity = await recordLegacyEmbeddingSource( + root, + { + sourceFile, + schemaFingerprint: 'schema', + recordedIdentity: knownProvider ? identity : { dimensions: 4 }, + }, + unsafe, + ); + let generation = 1; + const graphFile = () => 'lbug.generation.' + String(generation).padStart(32, '0'); + const graphGeneration = () => ({ + version: 1 as const, + id: String(generation).padStart(32, '0'), + coveredContent: 'b'.repeat(64), + }); + const meta: RepoMeta = { + repoPath: root, + lastCommit: 'commit', + indexedAt: '2026-11-04T00:00:00Z', + graphPath: graphFile(), + graphGeneration: graphGeneration(), + legacyVectorSource: legacyIdentity, + schemaFingerprint: 'schema', + parseCacheIdentity: { base: 'parser' }, + scopeExtractionReceipt: 1, + runnerIdentity: { + schemaVersion: 4, + cliVersion: 'test', + runtime: { + executablePath: process.execPath, + version: process.version, + platform: process.platform, + architecture: process.arch, + modulesAbi: '127', + libc: 'glibc', + }, + invokedArtifact: { path: 'test', digest: 'test' }, + build: { + kind: 'distribution', + rootPath: root, + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'build', + }, + dependencyRuntime: { + manifestPath: 'package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'runtime', + }, + }, + }; + await fs.writeFile(path.join(root, graphFile()), 'graph opened by the deterministic test source'); + await saveMeta(root, meta); + const changed = new Map(); + const events: string[] = []; + const provider = vi.fn(async (texts: string[]) => { + events.push('provider'); + return texts.map(() => new Float32Array([1, 0, 0, 0])); + }); + const initializeProvider = vi.fn(async () => {}); + const source = { + scan: async function* (label: string) { + if (label === 'Const') + for (let i = 0; i < count; i++) + yield { + id: id(i), + contentBytes: Buffer.byteLength(changed.get(id(i)) ?? node(id(i)).content), + metadataBytes: 1024, + }; + }, + fetch: async (_label: string, ids: readonly string[]) => + ids.map((key) => ({ ...node(key), content: changed.get(key) ?? node(key).content })), + close: async () => {}, + }; + const options = (): PublishedEmbeddingJobOptions => ({ + metadataDirectory: root, + graphPath: path.join(root, graphFile()), + config, + resources: { + graphPoolBytes: 16 * 1024 * 1024, + vectorPoolBytes: 64 * 1024 * 1024, + queueBytes: 16 * 1024 * 1024, + nativeThreads: 1, + modelBytes: 0, + modelHeapMb: 0, + reservedBytes: 96 * 1024 * 1024, + }, + metadata: { + loadMetadata: () => loadMeta(root), + saveMetadata: (value) => saveMeta(root, value), + }, + log: (message) => events.push(message), + }); + const run = (extra: Partial = {}) => + runPublishedEmbeddingJob( + { ...options(), ...extra }, + { identity: () => identity, openSource: async () => source, provider, initializeProvider }, + ); + const nextGraph = async () => { + const previous = await loadMeta(root); + if (!previous) throw new Error('missing metadata'); + generation++; + await fs.writeFile(path.join(root, graphFile()), 'new graph'); + await saveMeta(root, { + ...previous, + graphPath: graphFile(), + graphGeneration: graphGeneration(), + legacyVectorSource: undefined, + }); + }; + return { + root, + run, + options, + nextGraph, + provider, + initializeProvider, + changed, + events, + id, + sourcePath, + legacyIdentity, + }; +} + +describe('legacy migration through the published job controller', () => { + it('keeps provider batches full when legacy groups are ineligible for reuse', async () => { + const f = await fixture( + 16, + Array.from({ length: 16 }, (_, i) => `Const:a.ts:n${String(i).padStart(5, '0')}`), + ); + expect((await f.run()).phase).toBe('ready'); + expect(f.provider.mock.calls.map(([texts]) => texts.length)).toEqual([8, 8]); + }); + it('retains one legacy source and pending store across two crashes and graph changes with zero paid replay', async () => { + const f = await fixture(); + const before = await fs.stat(f.sourcePath); + let scans = 0; + const prepare = VectorStore.prototype.prepareLegacyReuseIndex; + vi.spyOn(VectorStore.prototype, 'prepareLegacyReuseIndex').mockImplementation(function ( + this: VectorStore, + source, + groups, + excluded, + signal, + ) { + return prepare.call( + this, + source, + async function* () { + scans++; + yield* groups(); + }, + excluded, + signal, + ); + }); + await expect( + f.run({ + onProgress: (state) => { + if (state.reusedNodes === 64) throw new Error('first stop'); + }, + }), + ).rejects.toThrow('first stop'); + const first = (await loadMeta(f.root))?.embeddingJob; + expect(first?.artifactIdentity).toBeDefined(); + expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); + expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); + await f.nextGraph(); + await expect( + f.run({ + onProgress: (state) => { + if (state.reusedNodes === 128) throw new Error('second stop'); + }, + }), + ).rejects.toThrow('second stop'); + expect((await loadMeta(f.root))?.embeddingJob?.artifactIdentity).toBe(first?.artifactIdentity); + expect((await fs.stat(f.sourcePath)).ino).toBe(before.ino); + expect((await fs.stat(f.sourcePath)).mtimeMs).toBe(before.mtimeMs); + await f.nextGraph(); + const result = await f.run(); + expect(result.phase).toBe('ready'); + expect(result.state?.reusedNodes).toBe(130); + expect(result.metadata.vectorGeneration?.artifactIdentity).toBe(first?.artifactIdentity); + expect(scans).toBe(1); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + await expect(fs.stat(f.sourcePath)).rejects.toMatchObject({ code: 'ENOENT' }); + const owner = (await readEmbeddingJobRecord( + path.join(f.root, 'embeddings', 'spaces', first?.spaceKey ?? ''), + 'space.json', + )) as { owner: { fallbackLegacy?: string } }; + expect(owner.owner.fallbackLegacy).toBeUndefined(); + }); + it('regenerates only changed inputs and groups in the unsafe checkpoint window', async () => { + const f = await fixture(5, ['Const:a.ts:n00002']); + f.changed.set(f.id(1), 'const changed = 2;'); + const result = await f.run(); + expect(result.metadata.legacyVectorSource).toBeUndefined(); + expect((await loadMeta(f.root))?.legacyVectorSource).toBeUndefined(); + await expect(fs.stat(f.sourcePath)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(result.state?.reusedNodes).toBe(3); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(2); + expect(f.events[0]).toContain('chunk size/overlap'); + expect(f.events.indexOf('provider')).toBeGreaterThan(0); + }); + it('reports unknown provider provenance before requests and retains its source without opening it', async () => { + const f = await fixture(3, [], false); + const open = vi.spyOn(LegacyVectorSource, 'open'); + expect((await f.run()).phase).toBe('ready'); + expect(open).not.toHaveBeenCalled(); + expect(f.events[0]).toContain('provider/model/dimensions provenance is unknown'); + expect(f.events.indexOf('provider')).toBeGreaterThan(0); + expect((await fs.stat(f.sourcePath)).isFile()).toBe(true); + expect(f.provider.mock.calls.flatMap(([texts]) => texts)).toHaveLength(3); + }); + it('rejects changed receipts before model initialization or requests', async () => { + const f = await fixture(3); + await fs.writeFile( + path.join(f.root, 'embeddings', 'legacy', f.legacyIdentity, 'legacy.json'), + '{}', + ); + await expect(f.run()).rejects.toThrow('Malformed'); + expect(f.provider).not.toHaveBeenCalled(); + expect(f.initializeProvider).not.toHaveBeenCalled(); + }); + it('rejects a hard-linked live graph as a legacy source before paid work', async () => { + const f = await fixture(3); + const graph = f.options().graphPath; + await fs.unlink(graph); + await fs.link(f.sourcePath, graph); + // Refresh the fixture receipt after link creation changes ctime, so this + // still exercises the separate live-graph alias guard. + const legacyVectorSource = await recordLegacyEmbeddingSource( + f.root, + { + sourceFile: path.basename(f.sourcePath), + schemaFingerprint: 'schema', + recordedIdentity: { provider: 'local', model: 'test', dimensions: 4 }, + }, + [], + ); + const metadata = await loadMeta(f.root); + if (!metadata) throw new Error('Missing fixture metadata'); + await saveMeta(f.root, { ...metadata, legacyVectorSource }); + await expect(f.run()).rejects.toThrow('distinct from the mutable graph'); + expect(f.provider).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/integration/legacy-vector-export.test.ts b/gitnexus/test/integration/legacy-vector-export.test.ts new file mode 100644 index 000000000..db3a6e2f3 --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-export.test.ts @@ -0,0 +1,265 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, expect, it, vi } from 'vitest'; +import { exportLegacyVectors } from '../../src/core/lbug/legacy-vector-export.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { LegacyVectorSource } from '../../src/core/lbug/legacy-vector-source.js'; +import { VectorStore } from '../../src/core/lbug/vector-store.js'; +import { readLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; + +const roots: string[] = []; +const pool = 32 * 1024 * 1024; +const identity = { provider: 'local', model: 'historical-model', dimensions: 4 }; +const config = resolveEmbeddingConfig({ threads: 1 }); +const node: EmbeddableNode = { + id: 'Const:a.ts:value', + name: 'value', + label: 'Const', + filePath: 'a.ts', + content: 'const value = 1;', + startLine: 1, + endLine: 1, +}; +afterEach(async () => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture(rows = 1, prefix = "gn-legacy-export-'") { + const root = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + roots.push(root); + const file = path.join(root, 'lbug'); + const handle = await openLbugConnection(lbug, file, { bufferPoolBytes: pool, threads: 1 }); + const query = async (sql: string): Promise[]> => { + const result = await handle.conn.query(sql); + const cursors = Array.isArray(result) ? result : [result]; + try { + return await cursors[0].getAll(); + } finally { + for (const cursor of cursors) cursor.close(); + } + }; + await query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ); + await query('CREATE NODE TABLE File (id STRING, content STRING, PRIMARY KEY(id))'); + await query( + "CREATE (:File {id:'File:a.ts',content:'Graph content must stay out of the export'})", + ); + const hash = contentHashForNode(node, config); + if (rows) + await query( + `UNWIND range(0, ${rows - 1}) AS i CREATE (:CodeEmbedding {id:'Const:a.ts:value:' + CAST(i AS STRING),nodeId:'Const:a.ts:value',chunkIndex:i,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'${hash}'})`, + ); + await query('CHECKPOINT'); + const options = { + metadataDirectory: root, + query, + provenance: { schemaFingerprint: 'schema', recordedIdentity: identity }, + excluded: [] as string[], + bufferPoolBytes: pool, + threads: 1, + }; + return { + root, + file, + query, + hash, + options, + close: () => closeLbugConnection(handle, { strict: true, databasePath: file }), + }; +} + +it.each(["gn-legacy-export-'", "gn-legacy-export-路径-'"])( + 'exports a reusable vector table under %s without provider work', + async (prefix) => { + const f = await fixture(1, prefix); + const before = await fs.stat(f.file, { bigint: true }); + const copy = vi.spyOn(fs, 'copyFile'); + let store: VectorStore | undefined, source: LegacyVectorSource | undefined; + try { + const id = await exportLegacyVectors(f.options); + expect(id).toMatch(/^[a-f0-9]{64}$/); + const receipt = await readLegacyEmbeddingSource(f.root, id!); + expect(receipt.recordedIdentity).toEqual(identity); + expect(copy).not.toHaveBeenCalled(); + expect((await fs.stat(f.file, { bigint: true })).ino).toBe(before.ino); + expect(await f.query('MATCH (n:File) RETURN n.content AS content')).toEqual([ + { content: 'Graph content must stay out of the export' }, + ]); + const exported = await openLbugConnection(lbug, receipt.path, { + readOnly: true, + bufferPoolBytes: pool, + threads: 1, + }); + try { + const result = await exported.conn.query('CALL show_tables() RETURN name'); + const cursor = Array.isArray(result) ? result[0] : result; + try { + expect(await cursor.getAll()).toEqual([{ name: 'CodeEmbedding' }]); + } finally { + cursor.close(); + } + } finally { + await closeLbugConnection(exported, { strict: true, databasePath: receipt.path }); + } + await fs.mkdir(path.join(f.root, 'reuse')); + store = await VectorStore.open({ + directory: path.join(f.root, 'reuse'), + identity: 'b'.repeat(64), + dimensions: 4, + mode: 'create', + bufferPoolBytes: pool, + threads: 1, + batchBytes: 65536, + }); + source = await LegacyVectorSource.open( + { + path: receipt.path, + sourceIdentity: receipt.identity, + recordedIdentity: receipt.recordedIdentity, + expectedIdentity: identity, + expectedInputIdentity: 'c'.repeat(64), + config, + bufferPoolBytes: pool, + threads: 1, + batchBytes: 65536, + }, + store, + receipt.excludedIds, + ); + const group = await source.reusable(node.id, f.hash, { node }); + expect(group?.chunks).toBe(1); + expect(await source.readChunks(group!, 0, 1)).toEqual([ + { chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0, 0, 0] }, + ]); + } finally { + await source?.close(); + await store?.close(); + await f.close(); + } + }, +); + +it('keeps vector payloads out of JavaScript while transferring a larger table', async () => { + const f = await fixture(4096); + let largestResult = 0; + try { + const id = await exportLegacyVectors({ + ...f.options, + query: async (sql) => { + const rows = await f.query(sql); + largestResult = Math.max(largestResult, rows.length); + expect(rows.every((row) => !Object.hasOwn(row, 'embedding'))).toBe(true); + return rows; + }, + }); + expect(id).toBeDefined(); + expect(largestResult).toBe(7); + expect(await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count')).toEqual([ + { count: 4096 }, + ]); + } finally { + await f.close(); + } +}); + +it('does not invent missing historical provider or input provenance', async () => { + const f = await fixture(); + try { + const id = await exportLegacyVectors({ + ...f.options, + provenance: { schemaFingerprint: 'schema' }, + excluded: [node.id], + }); + const receipt = await readLegacyEmbeddingSource(f.root, id!); + expect(receipt.recordedIdentity).toBeUndefined(); + expect(receipt.recordedInputIdentity).toBeUndefined(); + const ids = []; + for await (const id of receipt.excludedIds()) ids.push(id); + expect(ids).toEqual([node.id]); + } finally { + await f.close(); + } +}); + +it('does not create an artifact for an empty vector table', async () => { + const f = await fixture(0); + try { + expect(await exportLegacyVectors(f.options)).toBeUndefined(); + expect( + (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), + ).toEqual([]); + } finally { + await f.close(); + } +}); + +it('rejects mismatched historical dimensions before exporting', async () => { + const f = await fixture(); + try { + await expect( + exportLegacyVectors({ + ...f.options, + provenance: { + schemaFingerprint: 'schema', + recordedIdentity: { ...identity, dimensions: 8 }, + }, + }), + ).rejects.toThrow('schema differs'); + expect( + (await fs.readdir(f.root)).filter((name) => name.startsWith('lbug.generation.')), + ).toEqual([]); + } finally { + await f.close(); + } +}); + +it('refuses publication when the source count changes during the transfer', async () => { + const f = await fixture(); + let counts = 0; + try { + await expect( + exportLegacyVectors({ + ...f.options, + query: async (sql) => { + if (sql.includes('count(e)') && ++counts === 2) return [{ count: 2 }]; + return f.query(sql); + }, + }), + ).rejects.toThrow('source changed'); + expect((await fs.readdir(f.root)).filter((name) => /generation|staging/.test(name))).toEqual( + [], + ); + expect((await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count'))[0].count).toBe(1); + } finally { + await f.close(); + } +}); + +it('cancels after the native export returns without publishing or mutating its source', async () => { + const f = await fixture(); + const abort = new AbortController(); + try { + await expect( + exportLegacyVectors({ + ...f.options, + signal: abort.signal, + query: async (sql) => { + const result = await f.query(sql); + if (sql.startsWith('COPY')) abort.abort(new Error('cancel migration')); + return result; + }, + }), + ).rejects.toThrow('cancel migration'); + expect((await fs.readdir(f.root)).filter((name) => /generation|\.csv$/.test(name))).toEqual([]); + expect((await f.query('MATCH (e:CodeEmbedding) RETURN count(e) AS count'))[0].count).toBe(1); + } finally { + await f.close(); + } +}); diff --git a/gitnexus/test/integration/legacy-vector-reuse-index.test.ts b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts new file mode 100644 index 000000000..752b8fef7 --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-reuse-index.test.ts @@ -0,0 +1,137 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type LegacyVectorGroup } from '../../src/core/lbug/vector-store.js'; + +const source = 'a'.repeat(64); +const roots: string[] = []; +const handles: VectorStore[] = []; +async function fixture() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-index-')); + roots.push(directory); + const options = { + directory, + identity: 'b'.repeat(64), + dimensions: 4, + // Match the production vector pool; LadybugDB 0.22 needs more than 8 MiB + // for the receipt tables and bounded insert batches. + bufferPoolBytes: 64 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + }; + const store = await VectorStore.open({ ...options, mode: 'create' }); + handles.push(store); + return { store, options }; +} +const group = (n: number): LegacyVectorGroup => ({ + nodeId: `Function:a.ts:n${n}`, + contentHash: `hash-${n}`, + chunks: 2, + sourceStartLine: n * 3, + sourceEndLine: n * 3 + 2, +}); +async function* groups(n: number) { + for (let i = 0; i < n; i++) yield group(i); +} +afterEach(async () => { + for (const handle of handles.splice(0)) await handle.close(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('bounded native legacy receipt index', () => { + it('streams receipt batches, excludes unsafe groups, and resumes without rescanning', async () => { + const { store, options } = await fixture(); + const scan = vi.fn(() => groups(1000)); + const excluded = vi.fn(() => [group(2).nodeId, group(999).nodeId]); + await store.prepareLegacyReuseIndex(source, scan, excluded); + expect(await store.legacyReuseGroup(source, group(998).nodeId)).toEqual(group(998)); + expect(await store.legacyReuseGroup(source, group(2).nodeId)).toBeNull(); + expect(await store.legacyReuseGroup('c'.repeat(64), group(998).nodeId)).toBeNull(); + expect(await store.completedNodes([group(998).nodeId])).toEqual(new Map()); + await store.close(); + const reopened = await VectorStore.open({ ...options, mode: 'resume' }); + handles.push(reopened); + await reopened.prepareLegacyReuseIndex(source, scan, excluded); + expect(scan).toHaveBeenCalledOnce(); + expect(excluded).toHaveBeenCalledOnce(); + expect(await reopened.legacyReuseGroup(source, group(998).nodeId)).toEqual(group(998)); + }); + it('rebuilds an interrupted receipt pass without deleting completed paid vectors', async () => { + const { store } = await fixture(); + const id = 'Function:a.ts:paid'; + await store.beginNode(id, 'paid-hash'); + await store.append(id, 'paid-hash', [ + { chunkIndex: 0, startLine: 1, endLine: 2, embedding: [1, 0, 0, 0] }, + ]); + await store.finishNode(id, 'paid-hash', 1); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield* groups(65); + throw new Error('interrupted'); + }, + () => [], + ), + ).rejects.toThrow('interrupted'); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await store.prepareLegacyReuseIndex( + source, + () => groups(3), + () => [], + ); + expect(await store.legacyReuseGroup(source, group(2).nodeId)).toEqual(group(2)); + expect(await store.legacyReuseGroup(source, group(64).nodeId)).toBeNull(); + expect((await store.completedNodes([id])).size).toBe(1); + }); + it('rejects duplicate or oversized receipts before marking the source complete', async () => { + const { store } = await fixture(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield group(0); + yield group(0); + }, + () => [], + ), + ).rejects.toThrow(); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield { ...group(1), contentHash: 'x'.repeat(1024 * 1024) }; + }, + () => [], + ), + ).rejects.toThrow('oversized'); + expect(await store.legacyReuseGroup(source, group(1).nodeId)).toBeNull(); + }); + it('keeps cancellation incomplete and refuses writes to a published artifact', async () => { + const { store } = await fixture(); + const controller = new AbortController(); + await expect( + store.prepareLegacyReuseIndex( + source, + async function* () { + yield* groups(65); + controller.abort(); + yield group(66); + }, + () => [], + controller.signal, + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(await store.legacyReuseGroup(source, group(0).nodeId)).toBeNull(); + await store.seal(0, 0); + await expect( + store.prepareLegacyReuseIndex( + source, + () => groups(1), + () => [], + ), + ).rejects.toThrow(); + }); +}); diff --git a/gitnexus/test/integration/legacy-vector-source.test.ts b/gitnexus/test/integration/legacy-vector-source.test.ts new file mode 100644 index 000000000..aae894a31 --- /dev/null +++ b/gitnexus/test/integration/legacy-vector-source.test.ts @@ -0,0 +1,313 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { VectorStore, type StoredVectorChunk } from '../../src/core/lbug/vector-store.js'; +import { + LegacyVectorSource, + legacyVectorCompatibility, + type LegacyVectorSourceOptions, +} from '../../src/core/lbug/legacy-vector-source.js'; +import { openLbugConnection, closeLbugConnection } from '../../src/core/lbug/lbug-config.js'; +import { chunkNode } from '../../src/core/embeddings/chunker.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { + generateEmbeddingText, + splitEmbeddingText, +} from '../../src/core/embeddings/text-generator.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; +import { isShortLabel, type EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { createEmbeddingWorkManifest } from '../../src/core/embeddings/embedding-work-manifest.js'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import { runEmbeddingJob } from '../../src/core/embeddings/embedding-job-runner.js'; + +const roots: string[] = []; +const handles: Array<{ close(): Promise }> = []; +const identity = { provider: 'local', model: 'recorded', dimensions: 4 }; +const config = resolveEmbeddingConfig({ chunkSize: 200, overlap: 20, maxInputChars: 900 }); +const node = (kind: 'File' | 'Const', content = 'const n = 1;'): EmbeddableNode => ({ + id: kind === 'File' ? 'File:a.txt' : 'Const:a.txt:n', + name: 'n', + label: kind, + filePath: 'a.txt', + content, + startLine: 1, + endLine: content.split('\n').length, +}); +async function legacyChunks(input: EmbeddableNode): Promise { + const chunks = isShortLabel(input.label) + ? [ + { + text: input.content, + chunkIndex: 0, + startLine: input.startLine ?? 0, + endLine: input.endLine ?? 0, + }, + ] + : await chunkNode( + input.label, + input.content, + input.filePath, + input.startLine ?? 0, + input.endLine ?? 0, + config.chunkSize, + config.overlap, + ); + const rows: StoredVectorChunk[] = []; + let tail = ''; + for (const chunk of chunks) { + const text = generateEmbeddingText(input, chunk.text, config, chunk.chunkIndex, tail); + for (const _part of splitEmbeddingText(text, config.maxInputChars)) + rows.push({ + chunkIndex: rows.length, + startLine: chunk.startLine, + endLine: chunk.endLine, + embedding: [1, 0, 0, 0], + }); + tail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : ''; + } + return rows; +} +async function fixture( + input: EmbeddableNode, + adjust?: (rows: StoredVectorChunk[]) => StoredVectorChunk[], +) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-source-')); + roots.push(root); + const dbPath = path.join(root, 'lbug'); + const handle = await openLbugConnection(lbug, dbPath, { + bufferPoolBytes: 8 * 1024 * 1024, + threads: 1, + }); + const close = (result: lbug.QueryResult | lbug.QueryResult[]) => { + for (const part of Array.isArray(result) ? result : [result]) part.close(); + }; + const hash = contentHashForNode(input, config); + const chunks = await legacyChunks(input); + try { + close( + await handle.conn.query( + 'CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[4], contentHash STRING, PRIMARY KEY(id))', + ), + ); + const insert = await handle.conn.prepare( + 'CREATE (:CodeEmbedding {id:$id,nodeId:$nodeId,chunkIndex:$chunkIndex,startLine:$startLine,endLine:$endLine,embedding:$embedding,contentHash:$contentHash})', + ); + for (const chunk of adjust ? adjust(chunks) : chunks) + close( + await handle.conn.execute(insert, { + id: `${input.id}:${chunk.chunkIndex}`, + nodeId: input.id, + contentHash: hash, + ...chunk, + }), + ); + close(await handle.conn.query('CHECKPOINT')); + } finally { + await closeLbugConnection(handle, { strict: true, databasePath: dbPath }); + } + const directory = path.join(root, 'pending'); + await fs.mkdir(directory); + const store = await VectorStore.open({ + directory, + identity: 'b'.repeat(64), + dimensions: 4, + bufferPoolBytes: 32 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + mode: 'create', + }); + handles.push(store); + const options: LegacyVectorSourceOptions = { + path: dbPath, + sourceIdentity: 'a'.repeat(64), + recordedIdentity: identity, + expectedIdentity: identity, + recordedInputIdentity: 'c'.repeat(64), + expectedInputIdentity: 'c'.repeat(64), + config, + bufferPoolBytes: 8 * 1024 * 1024, + threads: 1, + batchBytes: 1024 * 1024, + }; + const open = async ( + override: Partial = {}, + excluded: string[] = [], + ) => { + const source = await LegacyVectorSource.open( + { ...options, ...override }, + store, + () => excluded, + ); + handles.push(source); + return source; + }; + return { root, directory, store, options, open, hash, chunks }; +} +afterEach(async () => { + for (const handle of handles.splice(0).reverse()) await handle.close(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('verified legacy vector reuse', () => { + it('builds a legacy reuse catalog above 50000 rows with admitted reader and writer pools', async () => { + const f = await fixture(node('Const')); + const budget = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + const pool = Math.floor(budget.embedding.vectorPoolBytes / 2); + const db = await openLbugConnection(lbug, f.options.path, { + bufferPoolBytes: pool, + threads: 1, + }); + try { + const result = await db.conn.query(`UNWIND range(1, 50000) AS i CREATE (:CodeEmbedding { + id:'Const:large.ts:n' + CAST(i AS STRING) + ':0', nodeId:'Const:large.ts:n' + CAST(i AS STRING), + chunkIndex:0,startLine:1,endLine:1,embedding:[1,0,0,0],contentHash:'hash'})`); + for (const cursor of Array.isArray(result) ? result : [result]) cursor.close(); + } finally { + await closeLbugConnection(db, { strict: true, databasePath: f.options.path }); + } + await f.store.close(); + const index = await VectorStore.open({ + ...f.store.options, + bufferPoolBytes: pool, + mode: 'resume', + }); + handles.push(index); + const source = await LegacyVectorSource.open( + { ...f.options, bufferPoolBytes: pool }, + index, + () => [], + ); + handles.push(source); + for (const [nodeId, contentHash] of [ + [node('Const').id, f.hash], + ['Const:large.ts:n1', 'hash'], + ['Const:large.ts:n50000', 'hash'], + ]) + expect(await index.legacyReuseGroup(f.options.sourceIdentity, nodeId)).toMatchObject({ + nodeId, + contentHash, + chunks: 1, + }); + expect( + await index.legacyReuseGroup(f.options.sourceIdentity, 'Const:large.ts:n50001'), + ).toBeNull(); + }, 120_000); + + it('relocates chunk spans without treating the last chunk as the whole node range', async () => { + const original = { + ...node('File', Array.from({ length: 80 }, (_, i) => `const v${i} = ${i};`).join('\n')), + endLine: 200, + }; + const f = await fixture(original); + expect(Math.max(...f.chunks.map((chunk) => chunk.endLine))).toBeLessThan(original.endLine); + const reuse = await f.open(); + const moved = { ...original, startLine: 11, endLine: 210 }; + expect(await reuse.reusable(moved.id, f.hash, { node: moved })).toMatchObject({ + complete: true, + sourceStartLine: 1, + sourceEndLine: undefined, + }); + }); + it('rejects missing or different provider provenance before touching the database', async () => { + expect(legacyVectorCompatibility(undefined, identity)).toContain('unknown'); + expect( + legacyVectorCompatibility({ ...identity, provider: 'http:' + 'a'.repeat(64) }, identity), + ).toContain('differ'); + expect(legacyVectorCompatibility(identity, identity)).toBeUndefined(); + const f = await fixture(node('Const')); + await expect(f.open({ path: '/missing', recordedIdentity: undefined })).rejects.toThrow( + 'provenance is unknown', + ); + await expect( + f.open({ path: '/missing', recordedInputIdentity: 'd'.repeat(64) }), + ).rejects.toThrow('input provenance differs'); + }); + it('validates a whole multi-chunk group, copies bounded batches, and relocates with zero provider calls', async () => { + const original = node( + 'File', + Array.from({ length: 80 }, (_, i) => `export const value${i} = ${i};`).join('\n'), + ); + const f = await fixture(original); + expect(f.chunks.length).toBeGreaterThan(1); + const reuse = await f.open(); + const moved = { ...original, startLine: 11, endLine: (original.endLine ?? 0) + 10 }; + const source = { + scan: async function* (label: string) { + if (label === 'File') + yield { + id: moved.id, + contentBytes: Buffer.byteLength(moved.content), + metadataBytes: 512, + }; + }, + fetch: async () => [moved], + }; + const manifest = await createEmbeddingWorkManifest(f.directory, 'e'.repeat(64), source); + const journal = await EmbeddingJobJournal.create(f.directory, 'b'.repeat(64), manifest); + const provider = vi.fn(async (texts: string[]) => + texts.map(() => new Float32Array([1, 0, 0, 0])), + ); + const result = await runEmbeddingJob({ + directory: f.directory, + manifest, + journal, + source, + store: f.store, + reuse, + provider, + config, + queueBytes: 8 * 1024 * 1024, + }); + expect(provider).not.toHaveBeenCalled(); + expect(result.state.reusedNodes).toBe(1); + const receipt = await f.store.reusable(moved.id, f.hash); + if (!receipt) throw new Error('Missing migrated receipt'); + expect(receipt.chunks).toBe(f.chunks.length); + expect(await f.store.readChunks(receipt, 0, Math.min(8, receipt.chunks))).toEqual( + f.chunks.slice(0, 8).map((chunk) => ({ + ...chunk, + startLine: chunk.startLine + 10, + endLine: chunk.endLine + 10, + })), + ); + }); + it('does not infer unknown multi-chunk settings from identical line spans', async () => { + const input = node('File', 'a'.repeat(800)); + const f = await fixture(input); + expect(f.chunks.length).toBeGreaterThan(1); + expect(new Set(f.chunks.map((c) => `${c.startLine}:${c.endLine}`)).size).toBe(1); + const log = vi.fn(); + const reuse = await f.open({ recordedInputIdentity: undefined, log }); + expect(await reuse.reusable(input.id, f.hash, { node: input })).toBeNull(); + expect(log).toHaveBeenCalledOnce(); + expect(log.mock.calls[0][0]).toContain('chunk size/overlap'); + }); + it('can prove an unchunked short-label input without inventing a historical input receipt', async () => { + const input = node('Const'); + const f = await fixture(input); + const reuse = await f.open({ recordedInputIdentity: undefined }); + expect(await reuse.reusable(input.id, f.hash, { node: input })).toMatchObject({ + chunks: f.chunks.length, + }); + expect(reuse.options.recordedInputIdentity).toBeUndefined(); + }); + it('rejects changed hashes, holes, and groups in an unsafe checkpoint window', async () => { + const input = node('File', 'x'.repeat(1000)); + const f = await fixture(input, (rows) => rows.filter((row) => row.chunkIndex !== 1)); + expect(await (await f.open()).reusable(input.id, f.hash, { node: input })).toBeNull(); + const intact = await fixture(input); + const excluded = await intact.open({}, [input.id]); + expect(await excluded.reusable(input.id, intact.hash, { node: input })).toBeNull(); + const changed = await fixture(node('Const')); + expect( + await (await changed.open()).reusable('Const:a.txt:n', 'different', { node: node('Const') }), + ).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/shared-store-adoption.test.ts b/gitnexus/test/integration/shared-store-adoption.test.ts index 9c7485f5e..bfeaa8d62 100644 --- a/gitnexus/test/integration/shared-store-adoption.test.ts +++ b/gitnexus/test/integration/shared-store-adoption.test.ts @@ -3,7 +3,7 @@ import { existsSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; import { readSharedStorePointer, resolveSharedStore, @@ -116,6 +116,28 @@ describe('shared store adoption and reporting (#3352)', () => { ); }, 240_000); + it.each(['flat', 'branch'])( + 'retains unfinished private %s analysis before shared-store adoption', + async (kind) => { + await legacyIndex(wt); + const local = path.join(wt, '.gitnexus'); + const meta = await loadMeta(local); + if (!meta) throw new Error('Missing local fixture metadata'); + const directory = kind === 'flat' ? local : path.join(local, 'branches', 'unfinished'); + await fs.mkdir(directory, { recursive: true }); + await saveMeta(directory, { + ...meta, + incrementalInProgress: { startedAt: Date.now(), toWriteCount: 1 }, + }); + await analyze(wt); + expect((await listRegisteredRepos()).find((entry) => entry.path === wt)?.storagePath).toBe( + local, + ); + expect(readSharedStorePointer(wt)).toBeNull(); + }, + 240_000, + ); + it('status reports the shared graph and the leftover index with its removal command', async () => { await legacyIndex(wt); await analyze(wt); diff --git a/gitnexus/test/unit/analysis-embedding-phase.test.ts b/gitnexus/test/unit/analysis-embedding-phase.test.ts new file mode 100644 index 000000000..1e40299dd --- /dev/null +++ b/gitnexus/test/unit/analysis-embedding-phase.test.ts @@ -0,0 +1,133 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; + +const mocks = vi.hoisted(() => ({ load: vi.fn(), save: vi.fn(), prepared: vi.fn(), run: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: mocks.load, + saveMeta: mocks.save, +})); +vi.mock('../../src/core/analysis-publication.js', () => ({ + readPreparedAnalysis: mocks.prepared, + beginInPlaceAnalysis: vi.fn(), + prepareAnalysisPublication: vi.fn(), + publishPreparedAnalysis: vi.fn(), +})); +vi.mock('../../src/core/embeddings/embedding-job.js', () => ({ + runPublishedEmbeddingJob: mocks.run, +})); +vi.mock('../../src/core/embeddings/http-client.js', () => ({ isHttpMode: () => true })); +vi.mock('../../src/core/embeddings/embedding-identity.js', () => ({ + resolveEmbeddingIdentity: () => ({ provider: 'http:test', model: 'fixture', dimensions: 4 }), +})); +import { runAnalysisEmbeddingPhase } from '../../src/core/analysis-embedding-phase.js'; + +const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 2048 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, +}); +const metadata = { + repoPath: '/repo', + lastCommit: 'a'.repeat(40), + indexedAt: '', + stats: { nodes: 3, embeddings: 1 }, + graphGeneration: { id: 'b'.repeat(32) }, + embeddingJob: { identity: 'retained' }, +}; +const options = () => ({ + metadataDirectory: '/repo/.gitnexus', + graphPath: '/repo/.gitnexus/graph.lbug', + resources, + callbacks: { onProgress: vi.fn(), onGraphPublished: vi.fn(async () => {}) }, +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.load.mockResolvedValue(metadata); + mocks.prepared.mockResolvedValue(undefined); + mocks.run.mockResolvedValue({ phase: 'ready', metadata }); +}); + +describe('shared embedding phase publication contract', () => { + it('awaits reader refresh before admitting provider work', async () => { + const input = options(); + let visible!: () => void; + input.callbacks.onGraphPublished.mockImplementationOnce( + () => + new Promise((resolve) => { + visible = resolve; + }), + ); + const pending = runAnalysisEmbeddingPhase(input); + await vi.waitFor(() => expect(input.callbacks.onGraphPublished).toHaveBeenCalledOnce()); + expect(mocks.run).not.toHaveBeenCalled(); + visible(); + await expect(pending).resolves.toBe(metadata); + expect(mocks.run).toHaveBeenCalledOnce(); + }); + + it('does not report incomplete vector work as complete or clear its durable metadata', async () => { + mocks.run.mockResolvedValue({ phase: 'embedding', metadata }); + const input = options(); + await expect(runAnalysisEmbeddingPhase(input)).rejects.toThrow('work remains incomplete'); + expect(mocks.save).not.toHaveBeenCalled(); + expect(input.callbacks.onProgress).toHaveBeenLastCalledWith( + 'embeddings', + 98, + expect.any(String), + expect.objectContaining({ graphPhase: 'ready', embeddingProgress: { phase: 'failed' } }), + ); + }); + + it('keeps the last measured count and pending work when provider or final verification fails', async () => { + mocks.run.mockRejectedValue(new Error('Could not verify persisted embedding count')); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow( + 'Could not verify persisted embedding count', + ); + expect(mocks.save).not.toHaveBeenCalled(); + expect(metadata.stats.embeddings).toBe(1); + expect(metadata.embeddingJob.identity).toBe('retained'); + }); + + it('rejects a superseded graph before reader refresh or paid work', async () => { + const input = options(); + await expect( + runAnalysisEmbeddingPhase({ ...input, expectedGeneration: 'c'.repeat(32) }), + ).rejects.toThrow('Graph generation changed'); + expect(mocks.run).not.toHaveBeenCalled(); + expect(input.callbacks.onGraphPublished).not.toHaveBeenCalled(); + }); + + it.each(['partial', 'unverified-count'])( + 'rejects a foreign %s checkpoint before provider work', + async (kind) => { + mocks.load.mockResolvedValue({ + ...metadata, + embeddingCheckpoint: { + provider: 'http:other', + model: 'other', + dimensions: 4, + kind, + at: '', + nodesProcessed: 1, + totalNodes: 3, + chunksProcessed: 1, + pendingNodeIds: ['n2'], + }, + }); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow( + /different provider|Cannot resume/, + ); + expect(mocks.run).not.toHaveBeenCalled(); + expect(mocks.save).not.toHaveBeenCalled(); + }, + ); + + it('refuses prepared or malformed graph receipts without touching embeddings', async () => { + mocks.prepared.mockResolvedValue({ id: 'a'.repeat(32) }); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow('awaits publication'); + mocks.prepared.mockRejectedValue(new Error('Malformed publication receipt')); + await expect(runAnalysisEmbeddingPhase(options())).rejects.toThrow('Malformed'); + expect(mocks.run).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/analysis-host-resources.test.ts b/gitnexus/test/unit/analysis-host-resources.test.ts new file mode 100644 index 000000000..a6dbf2bd7 --- /dev/null +++ b/gitnexus/test/unit/analysis-host-resources.test.ts @@ -0,0 +1,129 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { + analysisMayEmbed, + resolveHostAnalysisResources, +} from '../../src/core/analysis-host-resources.js'; +import { heapCapMbFor } from '../../src/core/ingestion/utils/effective-ram.js'; + +const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), placement: vi.fn() })); +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (original) => ({ + ...(await original()), + effectiveRamBytes: () => 32768 * 1024 ** 2, +})); +vi.mock('node:os', () => ({ default: { availableParallelism: () => 16 } })); +vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: mocks.loadMeta, + getStoragePaths: (repo: string, branch: string | null) => ({ + metaPath: `${repo}/index/${branch ?? 'workspace'}/gitnexus.json`, + }), +})); +vi.mock('../../src/storage/branch-index.js', () => ({ resolveBranchPlacement: mocks.placement })); + +beforeEach(() => { + vi.clearAllMocks(); + for (const key of [ + 'GITNEXUS_EMBEDDING_URL', + 'GITNEXUS_EMBEDDING_MODEL', + 'GITNEXUS_WORKER_POOL_SIZE', + 'GITNEXUS_WORKER_HEAP_MB', + 'GITNEXUS_EMBEDDING_THREADS', + 'GITNEXUS_LBUG_BUFFER_POOL_SIZE', + ]) + vi.stubEnv(key, undefined); + mocks.loadMeta.mockResolvedValue(null); + mocks.placement.mockResolvedValue({ branch: 'release' }); +}); +afterEach(() => vi.unstubAllEnvs()); + +it('reserves local model capacity only for invocations that may embed', () => { + expect(resolveHostAnalysisResources({ includeEmbeddings: false }).embedding.modelBytes).toBe(0); + expect(resolveHostAnalysisResources({ includeEmbeddings: true }).embedding.modelBytes).toBe( + 2048 * 1024 ** 2, + ); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'http://localhost:1234/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'remote'); + expect(resolveHostAnalysisResources({ includeEmbeddings: true }).embedding.modelBytes).toBe(0); +}); + +it('divides the host allowance across concurrent analyses', () => { + const one = resolveHostAnalysisResources({ includeEmbeddings: false }); + const four = resolveHostAnalysisResources({ includeEmbeddings: false, concurrency: 4 }); + expect(four.automatic).toBe(true); + expect(four.mainHeapMb).toBe(heapCapMbFor(8192 * 1024 ** 2)); + expect(four.parse.nativeThreads * 4).toBe(one.parse.nativeThreads); + expect(four.parse.workers).toBeLessThanOrEqual(3); +}); + +it('honors bounded worker, pool, and embedding-thread overrides', () => { + vi.stubEnv('GITNEXUS_WORKER_POOL_SIZE', '8'); + vi.stubEnv('GITNEXUS_WORKER_HEAP_MB', '512'); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(256 * 1024 ** 2)); + vi.stubEnv('GITNEXUS_EMBEDDING_THREADS', '8'); + const budget = resolveHostAnalysisResources({ includeEmbeddings: true }); + expect(budget.parse.workers).toBe(8); + expect(budget.parse.workerHeapMb).toBe(512); + expect(budget.parse.graphPoolBytes).toBe(256 * 1024 ** 2); + expect(budget.embedding.nativeThreads).toBe(8); +}); + +it('ignores embedding-only thread configuration for a graph-only invocation', () => { + vi.stubEnv('GITNEXUS_EMBEDDING_THREADS', 'not-a-number'); + expect(resolveHostAnalysisResources({ includeEmbeddings: false }).parse.workers).toBe(15); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: true })).toThrow( + 'GITNEXUS_EMBEDDING_THREADS', + ); +}); + +it.each(['0', ' 0 ', '0.0'])( + 'explains why the native default pool %s cannot be reserved', + (value) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', value); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'bounded analysis cannot reserve', + ); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'Unset GITNEXUS_LBUG_BUFFER_POOL_SIZE', + ); + }, +); + +it.each(['-1', '1.5', 'NaN'])( + 'rejects an unbounded or malformed native pool override %s', + (value) => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', value); + expect(() => resolveHostAnalysisResources({ includeEmbeddings: false })).toThrow( + 'positive safe integer', + ); + }, +); + +it('honors explicit heap-only overcommit but rejects a conflicting family ceiling', () => { + expect( + resolveHostAnalysisResources({ includeEmbeddings: false, mainHeapMb: 40000 }).mainHeapMb, + ).toBe(40000); + expect(() => + resolveHostAnalysisResources({ includeEmbeddings: false, mainHeapMb: 40000, totalMb: 24576 }), + ).toThrow('cannot admit'); +}); + +it.each([ + ['pending job', { embeddingJob: {} }], + ['pending checkpoint', { embeddingCheckpoint: {} }], + ['preserved vectors', { stats: { embeddings: 1 } }], +])( + 'reserves retry/forced model work for %s from the exact branch metadata', + async (_case, metadata) => { + mocks.loadMeta.mockResolvedValue(metadata); + expect(await analysisMayEmbed('/repo', { force: true, branch: 'release' })).toBe(true); + expect(mocks.placement).toHaveBeenCalledWith('/repo', 'release'); + expect(mocks.loadMeta).toHaveBeenCalledWith('/repo/index/release'); + }, +); + +it('keeps plain preservation free of model work and lets explicit drop abandon retries', async () => { + mocks.loadMeta.mockResolvedValue({ stats: { embeddings: 100 } }); + expect(await analysisMayEmbed('/repo', {})).toBe(false); + mocks.loadMeta.mockResolvedValue({ embeddingJob: {} }); + expect(await analysisMayEmbed('/repo', { dropEmbeddings: true })).toBe(false); + expect(await analysisMayEmbed('/repo', { dropEmbeddings: true, embeddings: true })).toBe(true); +}); diff --git a/gitnexus/test/unit/analysis-parse-cache.test.ts b/gitnexus/test/unit/analysis-parse-cache.test.ts new file mode 100644 index 000000000..4bba90e51 --- /dev/null +++ b/gitnexus/test/unit/analysis-parse-cache.test.ts @@ -0,0 +1,224 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { + loadAnalysisParseCache, + publishAnalysisParseCache, + stageAnalysisParseCache, +} from '../../src/core/analysis-parse-cache.js'; +import { + createColdParseRebuildDir, + emptyParseCache, + loadParseCache, + loadParseCacheChunk, + persistParseCacheChunk, + saveParseCache, + type ParseCache, +} from '../../src/storage/parse-cache.js'; +import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { TEST_PARSE_IDENTITY } from '../helpers/cache-identity.js'; + +const generation = 'e'.repeat(32); +const a = 'a'.repeat(64), + b = 'b'.repeat(64), + c = 'c'.repeat(64), + d = 'd'.repeat(64); +const result = (fileCount: number): ParseWorkerResult => ({ + nodes: [], + relationships: [], + symbols: [], + calls: [], + assignments: [], + routes: [], + fetchCalls: [], + fetchWrapperDefs: [], + decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + toolDefs: [], + ormQueries: [], + constructorBindings: [], + fileScopeBindings: [], + parsedFiles: [], + skippedLanguages: {}, + fileCount, +}); +let root: string, stage: string; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-cache-handoff-'))); + stage = await createColdParseRebuildDir(root); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); +const metadata = (keys: string[]): RepoMeta => ({ + repoPath: root, + lastCommit: '1'.repeat(40), + indexedAt: '2026-11-04T00:00:00Z', + graphGeneration: { version: 1, id: generation, coveredContent: 'f'.repeat(64) }, + parseCacheIdentity: { base: TEST_PARSE_IDENTITY.base }, + cacheKeys: keys, +}); +const options = (keys: string[] = [], complete = true) => ({ + storagePath: root, + metadataDirectory: root, + cacheRoot: root, + publishedGeneration: generation, + collectBranchKeys: async () => ({ keys: new Set(keys), complete }), + log: vi.fn(), +}); +async function live(keys: [string, number][]) { + const cache = emptyParseCache(TEST_PARSE_IDENTITY); + for (const [key, count] of keys) { + cache.usedKeys.add(key); + cache.entries.set(key, [result(count)]); + } + await saveParseCache(root, cache); +} +async function restored(meta: RepoMeta): Promise { + const cache = await loadAnalysisParseCache(root, meta); + if (!cache) throw new Error('Missing handoff fixture'); + return cache; +} +async function mutateReceipt(change: (payload: Record) => void) { + const file = path.join(root, `analysis-cache-${generation}.json`); + const envelope = JSON.parse(await fs.readFile(file, 'utf8')); + change(envelope.payload); + envelope.checksum = createHash('sha256').update(JSON.stringify(envelope.payload)).digest('hex'); + await fs.writeFile(file, JSON.stringify(envelope)); +} + +it('rejects a checksummed selection that repeats one key instead of including another', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys = new Set([a, b]); + await stageAnalysisParseCache(root, generation, cache); + await mutateReceipt((payload) => { + payload.used = [a, a]; + }); + await expect(loadAnalysisParseCache(root, metadata([a, b]))).rejects.toThrow('selection differs'); +}); + +it('hands off keys without copying live shards, then merges a newer sibling publication', async () => { + await live([ + [a, 1], + [b, 2], + ]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys = new Set([a, b, c]); + cache.staleKeys = new Set([b]); + await persistParseCacheChunk(cache, a, [result(11)]); + await persistParseCacheChunk(cache, c, [result(3)]); + const copy = vi.spyOn(fs, 'copyFile'); + await stageAnalysisParseCache(root, generation, cache); + expect(copy).not.toHaveBeenCalled(); + expect( + (await loadParseCacheChunk(await loadParseCache(root, TEST_PARSE_IDENTITY, stage), a))?.[0] + .fileCount, + ).toBe(1); + expect((await fs.readdir(path.join(stage, 'parse-cache'))).sort()).toEqual([ + `${a}.v8`, + `${c}.v8`, + ]); + // A sibling finishes after the graph child exited, before cache publication. + await live([ + [a, 1], + [b, 2], + [d, 4], + ]); + await publishAnalysisParseCache(await restored(metadata([a, b, c])), options([d])); + const published = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + expect([...(published.onDiskKeys ?? [])].sort()).toEqual([a, c, d]); + expect((await loadParseCacheChunk(published, a))?.[0].fileCount).toBe(11); + expect((await loadParseCacheChunk(published, c))?.[0].fileCount).toBe(3); + expect((await loadParseCacheChunk(published, d))?.[0].fileCount).toBe(4); + await expect(fs.stat(path.join(root, `analysis-cache-${generation}.json`))).rejects.toMatchObject( + { code: 'ENOENT' }, + ); +}); + +it('retains its receipt when cache publication fails', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + await persistParseCacheChunk(cache, a, [result(2)]); + await stageAnalysisParseCache(root, generation, cache); + vi.spyOn(fs, 'rename').mockRejectedValueOnce(new Error('cache publication interrupted')); + await expect(publishAnalysisParseCache(await restored(metadata([a])), options())).rejects.toThrow( + 'cache publication interrupted', + ); + await expect( + fs.stat(path.join(root, `analysis-cache-${generation}.json`)), + ).resolves.toBeDefined(); + expect((await loadParseCacheChunk(await restored(metadata([a])), a))?.[0].fileCount).toBe(2); +}); + +it('flushes memory fallback shards before the child can hand off its cache', async () => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + cache.entries.set(a, [result(123)]); + await stageAnalysisParseCache(root, generation, cache); + expect(cache.entries.size).toBe(0); + const recovered = await restored(metadata([a])); + expect(recovered.entries.size).toBe(0); + expect((await loadParseCacheChunk(recovered, a))?.[0].fileCount).toBe(123); + const receipt = await fs.readFile(path.join(root, `analysis-cache-${generation}.json`), 'utf8'); + expect(receipt).not.toContain('fileCount'); +}); + +it('does not revive the old shard when a staged repair disappears', async () => { + await live([[a, 1]]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys.add(a); + await persistParseCacheChunk(cache, a, [result(2)]); + await stageAnalysisParseCache(root, generation, cache); + await fs.unlink(path.join(stage, 'parse-cache', `${a}.v8`)); + await publishAnalysisParseCache(await restored(metadata([a])), options()); + expect( + await loadParseCacheChunk(await loadParseCache(root, TEST_PARSE_IDENTITY, stage), a), + ).toBeUndefined(); +}); + +it('retains sibling shards when metadata cannot be fully inspected, while dropping stale keys', async () => { + await live([ + [a, 1], + [b, 2], + [c, 3], + ]); + const cache = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + cache.usedKeys.add(a); + cache.staleKeys = new Set([b]); + await stageAnalysisParseCache(root, generation, cache); + await publishAnalysisParseCache(await restored(metadata([a])), options([], false)); + const published = await loadParseCache(root, TEST_PARSE_IDENTITY, stage); + expect([...(published.onDiskKeys ?? [])].sort()).toEqual([a, c]); +}); + +it.each(['generation', 'parser', 'selection', 'path', 'checksum'])( + 'refuses an incompatible or corrupt %s receipt', + async (kind) => { + const cache = emptyParseCache(TEST_PARSE_IDENTITY, { writeStoragePath: stage }); + cache.usedKeys.add(a); + await stageAnalysisParseCache(root, generation, cache); + if (kind === 'checksum') { + const file = path.join(root, `analysis-cache-${generation}.json`); + const envelope = JSON.parse(await fs.readFile(file, 'utf8')); + envelope.checksum = '0'.repeat(64); + await fs.writeFile(file, JSON.stringify(envelope)); + } else + await mutateReceipt((payload) => { + if (kind === 'generation') payload.generation = '9'.repeat(32); + if (kind === 'parser') payload.identity = { base: 'other', pdg: 'other' }; + if (kind === 'selection') payload.used = [b]; + if (kind === 'path') payload.stage = '../foreign'; + }); + await expect(loadAnalysisParseCache(root, metadata([a]))).rejects.toThrow(); + expect((await fs.stat(stage)).isDirectory()).toBe(true); + }, +); + +it('treats a missing optional cache receipt as absent', async () => { + expect(await loadAnalysisParseCache(root, metadata([]))).toBeUndefined(); +}); diff --git a/gitnexus/test/unit/analysis-phase-supervisor.test.ts b/gitnexus/test/unit/analysis-phase-supervisor.test.ts new file mode 100644 index 000000000..321e29749 --- /dev/null +++ b/gitnexus/test/unit/analysis-phase-supervisor.test.ts @@ -0,0 +1,295 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + projectPreparedAnalysisHandoff, + publishAnalysisAfterProcessExit, + type AnalysisPhasePublicationActions, + type PreparedAnalysisHandoff, +} from '../../src/core/analysis-phase-supervisor.js'; +import { waitForAnalysisProcessExit } from '../helpers/analysis-process-exit.js'; +import { + readPreparedAnalysis, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const roots: string[] = []; +const children: { child: ChildProcess; exited: Promise }[] = []; +afterEach(async () => { + for (const { child, exited } of children.splice(0)) { + if (child.connected) child.send({ exitCode: 0 }); + await exited.catch(() => {}); + } + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-phase-'))); + roots.push(root); + const repoPath = path.join(root, 'repo'); + await fs.mkdir(repoPath); + const commit = 'a'.repeat(40); + const metadata: RepoMeta = { + repoPath, + lastCommit: commit, + indexedAt: '2026-11-04T00:00:00Z', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }; + await saveMeta(root, metadata); + await fs.writeFile(path.join(root, 'lbug'), 'old graph'); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'new graph'); + const target = { + repoPath, + storagePath: root, + metadataDirectory: root, + repoName: 'fixture', + commit, + }; + const moduleUrl = new URL('../../dist/core/analysis-publication.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { prepareAnalysisPublication } from ${JSON.stringify(moduleUrl)}; + process.once('message', async ({ root, stage, metadata, target }) => { + const receipt = await prepareAnalysisPublication(root, stage, metadata, 'b'.repeat(64), { waitForProcessExit: true }); + process.once('message', ({ exitCode }) => { process.exitCode = exitCode; process.disconnect(); }); + process.send({ ...target, version: 1, publicationId: receipt.id }); + }); + `, + ], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ); + const exited = waitForAnalysisProcessExit(child); + // Keep rejection observed even when an assertion fails before awaiting it. + void exited.catch(() => {}); + children.push({ child, exited }); + const writerPid = child.pid; + if (writerPid === undefined) throw new Error('Graph fixture did not spawn'); + const prepared = new Promise((resolve) => + child.once('message', (value) => { + resolve(projectPreparedAnalysisHandoff(value)); + }), + ); + child.send({ root, stage, metadata, target }); + const handoff = await Promise.race([ + prepared, + exited.then(() => { + throw new Error('Writer exited before preparation'); + }), + ]); + const actions: AnalysisPhasePublicationActions = { + resolveTarget: vi.fn(async () => ({ ...target })), + saveMetadata: (value) => saveMeta(root, value), + verify: vi.fn(async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('new graph'); + }), + finalizeGraph: vi.fn(async (value) => value), + }; + const release = (exitCode = 0) => child.send({ exitCode }); + const finish = async () => { + await exited; + return publishAnalysisAfterProcessExit(handoff, writerPid, actions); + }; + return { root, child, writerPid, exited, handoff, metadata, target, actions, release, finish }; +} + +describe('graph process handoff', () => { + it('continues when a waiting writer already recovered the exact publication', async () => { + const f = await fixture(); + f.release(); + await f.exited; + const recovered = await publishPreparedAnalysis(f.root, f.actions); + expect(await readPreparedAnalysis(f.root)).toBeUndefined(); + expect((await f.finish()).graphGeneration).toEqual(recovered?.graphGeneration); + expect(f.actions.finalizeGraph).toHaveBeenCalledOnce(); + expect(f.actions.verify).toHaveBeenCalledOnce(); + }); + + it.each(['foreign', 'dirty', 'collapsed', 'commit'] as const)( + 'rejects a %s publication without a receipt', + async (state) => { + const f = await fixture(); + f.release(); + await f.exited; + const recovered = await publishPreparedAnalysis(f.root, f.actions); + if (!recovered) throw new Error('Missing publication'); + await saveMeta(f.root, { + ...recovered, + ...(state === 'foreign' + ? { graphGeneration: { ...recovered.graphGeneration!, id: 'e'.repeat(32) } } + : {}), + ...(state === 'dirty' ? { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } } : {}), + ...(state === 'collapsed' ? { graphWriteCollapsed: { expected: 10, persisted: 1 } } : {}), + ...(state === 'commit' ? { lastCommit: 'f'.repeat(40) } : {}), + } as RepoMeta); + await expect(f.finish()).rejects.toThrow('attempt changed'); + expect(f.actions.finalizeGraph).not.toHaveBeenCalled(); + }, + ); + it('keeps prepared nonterminal and publishes only after actual exit', async () => { + const f = await fixture(); + const provider = vi.fn(); + const finishing = f.finish().then((metadata) => { + provider(); + return metadata; + }); + await new Promise(setImmediate); + expect(f.child.exitCode).toBeNull(); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + f.release(); + const published = await finishing; + expect(published.graphGeneration?.id).toBe(f.handoff.publicationId); + expect(f.actions.finalizeGraph).toHaveBeenCalledOnce(); + expect(f.actions.resolveTarget).toHaveBeenCalledTimes(3); + expect(provider).toHaveBeenCalledOnce(); + }); + + it('rejects an early caller independently of its process-exit waiter', async () => { + const f = await fixture(); + await expect( + publishAnalysisAfterProcessExit(f.handoff, f.writerPid, f.actions), + ).rejects.toThrow('has not exited'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it('retains artifacts when the child exits unsuccessfully after preparing', async () => { + const f = await fixture(); + const finishing = f.finish(); + f.release(1); + await expect(finishing).rejects.toThrow('unsuccessfully'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it('waits for exit after cancellation and never admits a new phase', async () => { + const f = await fixture(); + const controller = new AbortController(); + f.actions.signal = controller.signal; + let settled = false; + const finishing = f.finish().finally(() => { + settled = true; + }); + controller.abort(new Error('cancelled')); + await new Promise(setImmediate); + expect(settled).toBe(false); + expect(f.child.exitCode).toBeNull(); + f.release(); + await expect(finishing).rejects.toThrow('cancelled'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }); + + it.each(['repoName', 'metadataDirectory', 'storagePath', 'commit'] as const)( + 'rechecks %s after waiting for the existing slot lock', + async (field) => { + const f = await fixture(); + const held = await acquireIndexLock(f.root, { sweep: false }); + let startedWaiting!: () => void; + const waiting = new Promise((resolve) => { + startedWaiting = resolve; + }); + f.actions.onWaitStart = startedWaiting; + const finishing = f.finish(); + try { + f.release(); + await waiting; + f.actions.resolveTarget = async () => ({ + ...f.target, + [field]: f.target[field] + '-changed', + }); + } finally { + held.release(); + } + await expect(finishing).rejects.toThrow('changed during'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(f.root)).toBeDefined(); + }, + ); + + it('rejects a superseding writer even when the target and commit did not change', async () => { + const f = await fixture(); + f.release(); + await f.exited; + await saveMeta(f.root, { ...f.metadata, indexedAt: 'newer writer' }); + await expect(f.finish()).rejects.toThrow('superseded'); + expect(f.actions.verify).not.toHaveBeenCalled(); + expect((await loadMeta(f.root))?.indexedAt).toBe('newer writer'); + }); + + it('rejects a different attempt or writer before opening native resources', async () => { + const f = await fixture(); + f.release(); + await f.exited; + await expect( + publishAnalysisAfterProcessExit( + { ...f.handoff, publicationId: 'c'.repeat(32) }, + f.writerPid, + f.actions, + ), + ).rejects.toThrow('attempt changed'); + await expect( + publishAnalysisAfterProcessExit(f.handoff, f.writerPid + 1, f.actions), + ).rejects.toThrow('attempt changed'); + expect(f.actions.verify).not.toHaveBeenCalled(); + }); + + it('preserves the published graph when finalization fails', async () => { + const f = await fixture(); + f.actions.finalizeGraph = async () => { + throw new Error('registry unavailable'); + }; + f.release(); + await expect(f.finish()).rejects.toThrow('registry unavailable'); + const metadata = await loadMeta(f.root); + expect(metadata?.graphGeneration?.id).toBe(f.handoff.publicationId); + const graphFile = metadata?.graphPath; + if (!graphFile) throw new Error('Published graph path is missing'); + expect(await fs.readFile(path.join(f.root, graphFile), 'utf8')).toBe('new graph'); + }); + + it('finishes graph discoverability if cancellation arrives after publication', async () => { + const f = await fixture(); + const controller = new AbortController(); + f.actions.signal = controller.signal; + const finalized = vi.fn(async (metadata: RepoMeta) => { + controller.abort(new Error('cancelled after publication')); + return metadata; + }); + f.actions.finalizeGraph = finalized; + f.release(); + await expect(f.finish()).rejects.toThrow('cancelled after publication'); + expect(finalized).toHaveBeenCalledOnce(); + expect((await loadMeta(f.root))?.graphGeneration?.id).toBe(f.handoff.publicationId); + }); + + it('projects a bounded allowlist without traversing a live graph', async () => { + const f = await fixture(); + const source = { + ...f.handoff, + get pipelineResult(): never { + throw new Error('materialized graph'); + }, + }; + const projected = projectPreparedAnalysisHandoff(source); + expect(projected).toEqual(f.handoff); + expect(JSON.stringify(projected).length).toBeLessThan(2048); + expect(() => + projectPreparedAnalysisHandoff({ ...f.handoff, repoName: 'x'.repeat(1025) }), + ).toThrow('repoName'); + expect(() => + projectPreparedAnalysisHandoff({ ...f.handoff, metadataDirectory: '../foreign' }), + ).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/analysis-publication.test.ts b/gitnexus/test/unit/analysis-publication.test.ts new file mode 100644 index 000000000..31ad737a0 --- /dev/null +++ b/gitnexus/test/unit/analysis-publication.test.ts @@ -0,0 +1,769 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import * as indexLock from '../../src/storage/index-lock.js'; +import { recordLegacyEmbeddingSource } from '../../src/storage/legacy-embedding-source.js'; +import { reclaimAnalysisArtifacts } from '../../src/storage/analysis-artifact-cleanup.js'; +import { adoptEmbeddingArtifact } from '../../src/storage/embedding-artifact.js'; +import { writeEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; +import { spawn } from 'node:child_process'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, + readPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { resolveGraphPath } from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const directories: string[] = []; +const content = 'a'.repeat(64); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abcdef', + indexedAt: '2026-11-03T00:00:00Z', + // Publication preserves the independently verified runner receipt verbatim. + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], +}; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publication-'))); + directories.push(root); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + await fs.writeFile(path.join(root, 'lbug'), 'previous graph'); + await saveMeta(root, metadata); + const verify = vi.fn(async (file: string) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }); + const actions = { verify, saveMetadata: (meta: RepoMeta) => saveMeta(root, meta) }; + return { root, stage, actions }; +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('durable graph publication', () => { + it('makes metadata durable before publishing the recovery receipt', async () => { + const { root, stage } = await fixture(); + const events: string[] = []; + const open = fs.open.bind(fs); + const rename = fs.rename.bind(fs); + const label = (file: string) => + file === root + ? 'directory' + : path.basename(file).includes('analysis-meta-') + ? 'metadata' + : 'receipt'; + vi.spyOn(fs, 'open').mockImplementation(async (...args) => { + const handle = await open(...args); + const sync = handle.sync.bind(handle); + vi.spyOn(handle, 'sync').mockImplementation(async () => { + await sync(); + events.push(`sync:${label(String(args[0]))}`); + }); + return handle; + }); + vi.spyOn(fs, 'rename').mockImplementation(async (...args) => { + await rename(...args); + events.push(`rename:${label(String(args[1]))}`); + }); + await prepareAnalysisPublication(root, stage, metadata, content); + const directorySync = process.platform === 'win32' ? [] : ['sync:directory']; + expect(events).toEqual([ + 'sync:metadata', + 'rename:metadata', + ...directorySync, + 'sync:receipt', + 'rename:receipt', + ...directorySync, + ]); + }); + + it('bounds superseded graph generations across repeated staged publications', async () => { + const { root, actions } = await fixture(); + for (let i = 0; i < 5; i++) { + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + await prepareAnalysisPublication(root, stage, (await loadMeta(root))!, content); + await publishPreparedAnalysis(root, actions); + } + const graphs = (await fs.readdir(root)).filter((name) => + /^lbug\.generation\.[a-f0-9]{32}$/.test(name), + ); + expect(graphs).toHaveLength(2); + expect(graphs).toContain((await loadMeta(root))?.graphPath); + }); + + it('reclaims attempt metadata orphaned before its publication receipt was written', async () => { + const { root, stage } = await fixture(); + const rename = fs.rename.bind(fs); + const interrupted = vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + if (path.basename(String(target)) === 'analysis-publication.json') + throw new Error('Interrupted before receipt'); + await rename(source, target); + }); + await expect(prepareAnalysisPublication(root, stage, metadata, content)).rejects.toThrow( + 'Interrupted before receipt', + ); + interrupted.mockRestore(); + const orphan = (await fs.readdir(root)).find((name) => + /^analysis-meta-[a-f0-9]{32}\.json$/.test(name), + ); + expect(orphan).toBeDefined(); + await fs.writeFile(path.join(root, 'analysis-meta-not-owned.json'), 'keep'); + await reclaimAnalysisArtifacts(root, metadata); + expect(await fs.readdir(root)).not.toContain(orphan); + expect(await fs.readFile(path.join(root, 'analysis-meta-not-owned.json'), 'utf8')).toBe('keep'); + expect(await loadMeta(root)).toEqual(metadata); + + const pending = await prepareAnalysisPublication(root, stage, metadata, content); + await reclaimAnalysisArtifacts(root, metadata); + expect(await readPreparedAnalysis(root)).toEqual(pending); + expect(await fs.readFile(path.join(root, pending.metadataFile), 'utf8')).toContain(content); + }); + + it('retains paid vectors when an ownership space was renamed', async () => { + const { root } = await fixture(); + const identity = 'a'.repeat(64); + const job = path.join(root, 'embeddings', identity); + await fs.mkdir(job, { recursive: true }); + await writeEmbeddingJobRecord(job, 'job.json', { version: 1, identity }); + await fs.writeFile(path.join(job, 'vectors'), 'paid vectors'); + const record = await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: identity, + jobIdentity: identity, + graphGeneration: 'd'.repeat(32), + sourceIdentity: 'e'.repeat(64), + manifestDigest: 'f'.repeat(64), + phase: 'pending', + revalidateInputs: false, + }, + async () => {}, + ); + const parent = path.join(root, 'embeddings', 'spaces'); + await fs.rename(path.join(parent, record.spaceKey), path.join(parent, 'renamed-space')); + const warn = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, warn); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Unexpected embedding space entry')); + expect(await fs.readFile(path.join(job, 'vectors'), 'utf8')).toBe('paid vectors'); + expect(await fs.readFile(path.join(job, 'job.json'), 'utf8')).toContain(identity); + }); + + it.each(['pending', 'sealed'] as const)( + 'reclaims unreferenced jobs and obsolete graphs for a %s model owner', + async (phase) => { + const { root } = await fixture(); + const current = 'a'.repeat(64), + fallback = 'b'.repeat(64), + orphan = 'c'.repeat(64); + const ownerGraph = path.join(root, `lbug.generation.${'f'.repeat(32)}`); + await fs.writeFile(ownerGraph, 'old owner graph'); + for (const id of [current, fallback, orphan]) { + const directory = path.join(root, 'embeddings', id); + await fs.mkdir(directory, { recursive: true }); + await writeEmbeddingJobRecord(directory, 'job.json', { version: 1, identity: id }); + await fs.writeFile(path.join(directory, 'vectors'), 'paid vectors'); + } + await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: current, + jobIdentity: current, + fallbackArtifact: fallback, + graphGeneration: 'f'.repeat(32), + sourceIdentity: '1'.repeat(64), + manifestDigest: '2'.repeat(64), + phase, + revalidateInputs: false, + }, + async () => {}, + ); + await reclaimAnalysisArtifacts(root, metadata); + if (phase === 'pending') + expect(await fs.readFile(ownerGraph, 'utf8')).toBe('old owner graph'); + else await expect(fs.stat(ownerGraph)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(path.join(root, 'embeddings', current, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + expect(await fs.readFile(path.join(root, 'embeddings', fallback, 'vectors'), 'utf8')).toBe( + 'paid vectors', + ); + await expect(fs.stat(path.join(root, 'embeddings', orphan))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }, + ); + + it.each(['job.json', 'artifact.json', 'retained.json'])( + 'retains %s until its busy vector file can be removed', + async (receiptName) => { + const { root } = await fixture(); + const identity = 'e'.repeat(64); + const job = path.join(root, 'embeddings', identity); + await fs.mkdir(job, { recursive: true }); + await writeEmbeddingJobRecord(job, receiptName, { version: 1, identity }); + const vectors = path.join(job, 'vectors'); + await fs.writeFile(vectors, 'obsolete vectors'); + const remove = fs.rm.bind(fs); + const busy = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === vectors) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + return remove(file, options); + }); + const warn = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, warn); + expect(await fs.readFile(path.join(job, receiptName), 'utf8')).toContain(identity); + expect(await fs.readFile(vectors, 'utf8')).toBe('obsolete vectors'); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('cleanup will retry')); + busy.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata); + await expect(fs.stat(job)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('keeps the only legacy vector source while a model-space job references it', async () => { + const { root } = await fixture(); + const sourceFile = `lbug.generation.${'c'.repeat(32)}`; + await fs.writeFile(path.join(root, sourceFile), 'paid legacy vectors'); + const legacy = await recordLegacyEmbeddingSource( + root, + { sourceFile, schemaFingerprint: 'fixture' }, + [], + ); + await adoptEmbeddingArtifact( + root, + { + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + schema: 'fixture', + }, + null, + { + artifactIdentity: 'a'.repeat(64), + jobIdentity: 'a'.repeat(64), + fallbackLegacy: legacy, + graphGeneration: 'f'.repeat(32), + sourceIdentity: '1'.repeat(64), + manifestDigest: '2'.repeat(64), + phase: 'pending', + revalidateInputs: false, + }, + async () => {}, + ); + await reclaimAnalysisArtifacts(root, metadata); + expect(await fs.readFile(path.join(root, sourceFile), 'utf8')).toBe('paid legacy vectors'); + }); + + it('retains everything when an ownership receipt is corrupt and retries busy files later', async () => { + const { root } = await fixture(); + const graph = path.join(root, `lbug.generation.${'b'.repeat(32)}`); + await fs.writeFile(graph, 'previous graph'); + const space = path.join(root, 'embeddings', 'spaces', 'a'.repeat(64)); + await fs.mkdir(space, { recursive: true }); + await fs.writeFile(path.join(space, 'space.json'), '{}'); + const log = vi.fn(); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + expect(await fs.readFile(graph, 'utf8')).toBe('previous graph'); + expect(log).toHaveBeenCalledWith(expect.stringContaining('ownership could not be verified')); + await fs.rm(space, { recursive: true }); + const remove = fs.rm.bind(fs); + const blocked = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === graph) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + await remove(file, options); + }); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + expect(await fs.readFile(graph, 'utf8')).toBe('previous graph'); + blocked.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata, undefined, log); + await expect(fs.stat(graph)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.each(['.wal', '.shadow'])( + 'retries a busy orphan %s after deleting its graph', + async (suffix) => { + const { root } = await fixture(); + const graph = path.join(root, `lbug.generation.${'b'.repeat(32)}`); + await fs.writeFile(graph, 'retired graph'); + await fs.writeFile(graph + suffix, 'busy sidecar'); + const remove = fs.rm.bind(fs); + const blocked = vi.spyOn(fs, 'rm').mockImplementation(async (file, options) => { + if (String(file) === graph + suffix) + throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + await remove(file, options); + }); + await reclaimAnalysisArtifacts(root, metadata, undefined, vi.fn()); + await expect(fs.stat(graph)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(graph + suffix, 'utf8')).toBe('busy sidecar'); + blocked.mockRestore(); + await reclaimAnalysisArtifacts(root, metadata); + await expect(fs.stat(graph + suffix)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('rejects a publication receipt replaced after its descriptor is opened', async () => { + const { root, stage } = await fixture(); + await prepareAnalysisPublication(root, stage, metadata, content); + const receipt = path.join(root, 'analysis-publication.json'); + const open = fs.open.bind(fs); + vi.spyOn(fs, 'open').mockImplementation(async (...args) => { + const handle = await open(...args); + if (String(args[0]) === receipt) { + await fs.rename(receipt, receipt + '.retained'); + await fs.writeFile(receipt, 'foreign replacement'); + } + return handle; + }); + await expect(readPreparedAnalysis(root)).rejects.toThrow('changed while opening'); + expect(await fs.readFile(receipt, 'utf8')).toBe('foreign replacement'); + }); + + it('recovers a prior process incarnation even when its PID is now this process', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + const payload = { ...receipt, writerInstance: 'f'.repeat(32) }; + await fs.writeFile( + path.join(root, 'analysis-publication.json'), + JSON.stringify({ + payload, + checksum: createHash('sha256').update(JSON.stringify(payload)).digest('hex'), + }), + ); + expect((await publishPreparedAnalysis(root, actions))?.graphGeneration?.id).toBe(receipt.id); + }); + + it('does not wait for an unrelated process that reused the writer PID', async () => { + const { root, stage, actions } = await fixture(); + const start = vi.spyOn(indexLock, 'readProcStartTime').mockReturnValue('123'); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + start.mockReturnValue('456'); + expect((await publishPreparedAnalysis(root, actions))?.graphGeneration?.id).toBe(receipt.id); + }); + + it('publishes through a symlinked parent while validating the real slot', async () => { + const { root, stage } = await fixture(); + const slot = path.join(root, 'slot'); + await fs.mkdir(slot); + await fs.rename(stage, path.join(slot, path.basename(stage))); + const alias = path.join(root, 'parent-alias'); + await fs.symlink(root, alias, process.platform === 'win32' ? 'junction' : 'dir'); + const aliasedSlot = path.join(alias, 'slot'); + await saveMeta(aliasedSlot, metadata); + const receipt = await prepareAnalysisPublication( + aliasedSlot, + path.join(aliasedSlot, path.basename(stage)), + metadata, + content, + ); + const published = await publishPreparedAnalysis(aliasedSlot, { + verify: async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }, + saveMetadata: (meta) => saveMeta(aliasedSlot, meta), + }); + expect(published?.graphGeneration?.id).toBe(receipt.id); + }); + + it('publishes only after the real preparing process exits', async () => { + const { root, stage, actions } = await fixture(); + const moduleUrl = new URL('../../dist/core/analysis-publication.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { prepareAnalysisPublication } from ${JSON.stringify(moduleUrl)}; + process.once('message', async ({ root, stage, metadata, content }) => { + await prepareAnalysisPublication(root, stage, metadata, content, { waitForProcessExit: true }); + process.once('message', () => process.disconnect()); + process.send('prepared'); + }); + `, + ], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ); + const exited = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', resolve); + }); + const prepared = new Promise((resolve) => child.once('message', resolve)); + try { + child.send({ root, stage, metadata, content }); + await Promise.race([ + prepared, + exited.then(() => { + throw new Error('Writer exited before preparation'); + }), + ]); + expect(child.exitCode).toBeNull(); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('has not exited'); + expect(actions.verify).not.toHaveBeenCalled(); + child.send('release'); + expect(await exited).toBe(0); + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration).toBeDefined(); + expect(actions.verify).toHaveBeenCalledOnce(); + } finally { + if (child.connected) child.send('release'); + await exited; + } + }); + + it('waits for an exit-only writer even after it has prepared the graph', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content, { + waitForProcessExit: true, + }); + expect(receipt.writerPid).toBe(process.pid); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('has not exited'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + const probe = vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('exited'), { code: 'ESRCH' }); + }); + await publishPreparedAnalysis(root, actions); + expect(probe).toHaveBeenCalledWith(process.pid, 0); + expect(actions.verify).toHaveBeenCalledOnce(); + }); + + it('retains a prepared graph when process liveness cannot be established', async () => { + const { root, stage, actions } = await fixture(); + await prepareAnalysisPublication(root, stage, metadata, content, { waitForProcessExit: true }); + vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('permission denied'), { code: 'EPERM' }); + }); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('permission denied'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + }); + + it('rejects a superseded attempt before verifying or moving its graph', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const newer = { ...metadata, lastCommit: 'newer-writer' }; + await saveMeta(root, newer); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('superseded'); + expect(actions.verify).not.toHaveBeenCalled(); + expect(await loadMeta(root)).toEqual(newer); + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + }); + + it('publishes a verified generation and keeps the previous reader artifact', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + expect(resolveGraphPath(root)).toBe(path.join(root, 'lbug')); + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration).toEqual({ + version: 1, + id: receipt.id, + coveredContent: content, + }); + expect(actions.verify).toHaveBeenCalledOnce(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + if (!receipt.legacyGraphFile) throw new Error('Expected a retained legacy graph'); + expect(await fs.readFile(path.join(root, receipt.legacyGraphFile), 'utf8')).toBe( + 'previous graph', + ); + await expect(fs.stat(path.join(root, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + expect(await publishPreparedAnalysis(root, actions)).toBeUndefined(); + expect(await fs.readdir(root)).not.toContain(receipt.metadataFile); + }); + + it.each(['before-rename', 'after-rename', 'after-metadata'] as const)( + 'recovers a crash %s and re-verifies the native artifact', + async (point) => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + if (point === 'before-rename') { + await expect( + publishPreparedAnalysis(root, { + ...actions, + verify: async () => { + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + } else { + await expect( + publishPreparedAnalysis(root, { + ...actions, + saveMetadata: async (meta) => { + if (point === 'after-metadata') await actions.saveMetadata(meta); + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + } + expect(await readPreparedAnalysis(root)).toEqual(receipt); + actions.verify.mockClear(); + await publishPreparedAnalysis(root, actions); + expect(actions.verify).toHaveBeenCalledOnce(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + }, + ); + + it('retains recovery evidence when the metadata writer fails to publish the exact receipt', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + await expect( + publishPreparedAnalysis(root, { ...actions, saveMetadata: async () => {} }), + ).rejects.toThrow('does not match'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + await publishPreparedAnalysis(root, actions); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + }); + + it('keeps a completed publication usable when redundant metadata cleanup fails', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const unlink = fs.unlink.bind(fs); + vi.spyOn(fs, 'unlink').mockImplementation(async (file) => { + if (String(file) === path.join(root, receipt.metadataFile)) throw new Error('cleanup failed'); + await unlink(file); + }); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('cleanup failed'); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + expect(resolveGraphPath(root)).toBe(path.join(root, receipt.graphFile)); + expect(await publishPreparedAnalysis(root, actions)).toBeUndefined(); + }); + + it.each(['receipt', 'metadata', 'graph'] as const)( + 'rejects corrupted %s without publishing', + async (artifact) => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + const file = + artifact === 'receipt' + ? 'analysis-publication.json' + : artifact === 'metadata' + ? receipt.metadataFile + : receipt.stagingFile; + await fs.writeFile(path.join(root, file), 'corrupt'); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow(); + expect(resolveGraphPath(root)).toBe(path.join(root, 'lbug')); + expect(await fs.readdir(root)).toContain('analysis-publication.json'); + }, + ); + + it('refuses missing or conflicting graph artifacts', async () => { + const { root, stage, actions } = await fixture(); + const receipt = await prepareAnalysisPublication(root, stage, metadata, content); + await fs.writeFile(path.join(root, receipt.graphFile), 'conflict'); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('conflicting'); + await fs.unlink(path.join(root, receipt.graphFile)); + await fs.unlink(stage); + await expect(publishPreparedAnalysis(root, actions)).rejects.toThrow('missing'); + expect(actions.verify).not.toHaveBeenCalled(); + }); + + it('rejects outside paths, symlinks and a second uncommitted publication', async () => { + const { root, stage } = await fixture(); + await expect( + prepareAnalysisPublication( + root, + path.join(root, '..', path.basename(stage)), + metadata, + content, + ), + ).rejects.toThrow('outside'); + const link = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.symlink(stage, link); + await expect(prepareAnalysisPublication(root, link, metadata, content)).rejects.toThrow( + 'regular file', + ); + await prepareAnalysisPublication(root, stage, metadata, content); + await expect(prepareAnalysisPublication(root, stage, metadata, content)).rejects.toThrow( + 'Recover', + ); + }); + + it('resolves private generations in branch slots and never falls back from a missing published file', async () => { + const { root } = await fixture(); + const slot = path.dirname(getStoragePaths('/repo', 'feature', root).metaPath); + const graphPath = `lbug.generation.${'b'.repeat(32)}`; + await saveMeta(slot, { ...metadata, graphPath }); + expect(getStoragePaths('/repo', 'feature', root).lbugPath).toBe(path.join(slot, graphPath)); + await fs.symlink(path.join(root, 'lbug'), path.join(slot, graphPath)); + expect(() => getStoragePaths('/repo', 'feature', root)).toThrow('regular file'); + }); +}); + +describe('logical in-place generations', () => { + it('invalidates readiness before writes and retains one physical file across generations', async () => { + const { root, actions } = await fixture(); + let previous = metadata; + let file = path.join(root, 'lbug'); + const before = await fs.stat(file); + const generations: string[] = []; + for (let iteration = 0; iteration < 3; iteration++) { + const dirty = await beginInPlaceAnalysis( + root, + { + ...previous, + incrementalInProgress: { startedAt: Date.now(), toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + expect((await loadMeta(root))?.graphGeneration).toBeUndefined(); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + expect((await loadMeta(root))?.incrementalInProgress?.attemptId).toBe(attemptId); + // Simulate the delta write only after the dirty stamp is durable. + await fs.writeFile(file, 'verified graph'); + const receipt = await prepareAnalysisPublication(root, file, previous, content, { + mode: 'in-place', + attemptId, + }); + const published = await publishPreparedAnalysis(root, actions); + if (!published) throw new Error('Expected a published generation'); + file = resolveGraphPath(root); + expect((await fs.stat(file)).ino).toBe(before.ino); + expect(receipt.legacyGraphFile).toBeUndefined(); + expect(published.incrementalInProgress).toBeUndefined(); + expect(published.graphGeneration?.id).toBe(attemptId); + generations.push(attemptId); + previous = published; + } + expect(new Set(generations).size).toBe(3); + expect( + (await fs.readdir(root)).filter((name) => /^lbug\.generation\.[a-f0-9]{32}$/.test(name)), + ).toHaveLength(1); + await expect(fs.stat(path.join(root, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.each(['before-rename', 'after-rename', 'after-metadata'] as const)( + 'recovers in-place publication %s without restoring a false old generation', + async (point) => { + const { root, actions } = await fixture(); + const dirty = await beginInPlaceAnalysis( + root, + { + ...metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + await fs.writeFile(path.join(root, 'lbug'), 'verified graph'); + const receipt = await prepareAnalysisPublication( + root, + path.join(root, 'lbug'), + metadata, + content, + { + mode: 'in-place', + attemptId, + }, + ); + await expect( + publishPreparedAnalysis(root, { + verify: + point === 'before-rename' + ? async () => { + throw new Error('crash'); + } + : actions.verify, + saveMetadata: async (meta) => { + if (point === 'after-metadata') await actions.saveMetadata(meta); + throw new Error('crash'); + }, + }), + ).rejects.toThrow('crash'); + expect(await readPreparedAnalysis(root)).toEqual(receipt); + if (point !== 'after-metadata') { + expect((await loadMeta(root))?.graphGeneration).toBeUndefined(); + expect((await loadMeta(root))?.incrementalInProgress?.attemptId).toBe(attemptId); + } + const published = await publishPreparedAnalysis(root, actions); + expect(published?.graphGeneration?.id).toBe(attemptId); + expect(published?.incrementalInProgress).toBeUndefined(); + }, + ); + + it('requires a durable matching attempt and rejects another writer before preparation', async () => { + const { root, actions } = await fixture(); + const file = path.join(root, 'lbug'); + await expect( + prepareAnalysisPublication(root, file, metadata, content, { + mode: 'in-place', + attemptId: 'a'.repeat(32), + }), + ).rejects.toThrow('missing or superseded'); + const dirty = await beginInPlaceAnalysis( + root, + { + ...metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + actions.saveMetadata, + ); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Expected an in-place attempt'); + await saveMeta(root, { ...metadata, lastCommit: 'newer' }); + await expect( + prepareAnalysisPublication(root, file, metadata, content, { + mode: 'in-place', + attemptId, + }), + ).rejects.toThrow('missing or superseded'); + expect(await readPreparedAnalysis(root)).toBeUndefined(); + }); + + it('rejects a shared pointer, missing dirty plan and an outstanding prepared publication', async () => { + const { root, stage, actions } = await fixture(); + await expect(beginInPlaceAnalysis(root, metadata, actions.saveMetadata)).rejects.toThrow( + 'dirty', + ); + const dirty = { ...metadata, incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }; + await expect( + beginInPlaceAnalysis(root, { ...dirty, graphPath: '/shared/lbug' }, actions.saveMetadata), + ).rejects.toThrow('private'); + await prepareAnalysisPublication(root, stage, metadata, content); + await expect(beginInPlaceAnalysis(root, dirty, actions.saveMetadata)).rejects.toThrow( + 'Recover', + ); + }); + + it('rejects an aliased target or foreign slot before changing its dirty marker', async () => { + const { root, stage, actions } = await fixture(); + const dirty = { ...metadata, incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }; + await expect( + beginInPlaceAnalysis(root, { ...dirty, repoPath: '/other-repo' }, actions.saveMetadata), + ).rejects.toThrow('another repository'); + await fs.unlink(path.join(root, 'lbug')); + await fs.symlink(stage, path.join(root, 'lbug')); + await expect(beginInPlaceAnalysis(root, dirty, actions.saveMetadata)).rejects.toThrow( + 'regular file', + ); + expect(await loadMeta(root)).toEqual(metadata); + }); +}); diff --git a/gitnexus/test/unit/analysis-resource-budget.test.ts b/gitnexus/test/unit/analysis-resource-budget.test.ts new file mode 100644 index 000000000..9f3fa0e78 --- /dev/null +++ b/gitnexus/test/unit/analysis-resource-budget.test.ts @@ -0,0 +1,282 @@ +import { heapCapMbFor } from '../../src/core/ingestion/utils/effective-ram.js'; +import { describe, expect, it } from 'vitest'; +import { + EmbeddingByteBudget, + resolveAnalysisResourceBudget, +} from '../../src/core/analysis-resource-budget.js'; + +const MIB = 1024 * 1024; +const request = { effectiveMemoryBytes: 32_768 * MIB, availableCpus: 24, modelMemoryMb: 2048 }; + +describe('analysis resource admission', () => { + it.each([1, 4, 16])( + 'preserves the automatic graph cap and explicit COPY floor at granule ratio %s', + (scale) => { + const automatic = resolveAnalysisResourceBudget({ ...request, graphPoolGranuleRatio: scale }); + expect(automatic.parse.graphPoolBytes).toBe( + Math.min(2048 * scale, Math.floor(32768 * 0.8)) * MIB, + ); + const explicit = resolveAnalysisResourceBudget({ + ...request, + totalMb: 16384, + graphPoolGranuleRatio: scale, + }); + expect(explicit.parse.graphPoolBytes).toBeGreaterThanOrEqual(256 * scale * MIB); + expect(explicit.parse.reservedBytes).toBeLessThanOrEqual(explicit.totalBytes); + }, + ); + it('rejects an explicit family budget that cannot fit the 64 KiB COPY floor', () => { + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 2048, graphPoolGranuleRatio: 16 }), + ).toThrow('cannot admit'); + }); + it.each([4096, 8192, 32768])( + 'preserves automatic main-heap and worker capacity on a %s MiB host', + (memoryMb) => { + const input = { + effectiveMemoryBytes: memoryMb * MIB, + availableCpus: 8, + preferredWorkers: 16, + modelMemoryMb: 0, + }; + const graph = resolveAnalysisResourceBudget(input); + const local = resolveAnalysisResourceBudget({ ...input, modelMemoryMb: 2048 }); + const pinned = resolveAnalysisResourceBudget({ + ...input, + mainHeapMb: 12000, + allowHeapOvercommit: true, + }); + expect(graph.mainHeapMb).toBe(heapCapMbFor(memoryMb * 1024 ** 2)); + expect(graph.parse.workers).toBe(7); + expect(graph.parse.workerHeapMb).toBe( + Math.min(4096, Math.max(512, Math.floor(memoryMb / 2 / 7))), + ); + expect(local.parse).toEqual(graph.parse); + expect(pinned.parse.workers).toBe(graph.parse.workers); + expect(pinned.parse.workerHeapMb).toBe(graph.parse.workerHeapMb); + expect(local.embedding.reservedBytes).toBeLessThanOrEqual(local.totalBytes); + expect(graph.embedding.vectorPoolBytes).toBeGreaterThan(64 * MIB); + }, + ); + it('preserves an explicit heap override without inflating automatic native allocations', () => { + const host = { ...request, effectiveMemoryBytes: 7500 * MIB, modelMemoryMb: 0 }; + const automatic = resolveAnalysisResourceBudget(host); + const budget = resolveAnalysisResourceBudget({ + ...host, + mainHeapMb: 12000, + allowHeapOvercommit: true, + }); + expect(budget.mainHeapMb).toBe(12000); + expect(budget.totalBytes).toBeGreaterThan(host.effectiveMemoryBytes); + expect(budget.parse.graphPoolBytes).toBe(automatic.parse.graphPoolBytes); + expect(budget.embedding.queueBytes).toBe(automatic.embedding.queueBytes); + expect(budget.headroomBytes).toBe(automatic.headroomBytes); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => resolveAnalysisResourceBudget({ ...host, mainHeapMb: 12000 })).toThrow( + 'cannot admit', + ); + expect(() => + resolveAnalysisResourceBudget({ + ...host, + mainHeapMb: 12000, + allowHeapOvercommit: true, + totalMb: 5625, + }), + ).toThrow('cannot admit'); + }); + + it('charges a retained supervisor once per phase, independently of worker thread count', () => { + const fixed = { + ...request, + totalMb: 24576, + mainHeapMb: 8192, + workers: 2, + workerHeapMb: 1024, + graphPoolBytes: 1024 * MIB, + }; + const before = resolveAnalysisResourceBudget(fixed); + const after = resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 192 * MIB + 1 }); + expect(after.supervisorBytes).toBe(193 * MIB); + expect(after.mainHeapMb).toBe(before.mainHeapMb); + expect(after.parse.workers).toBe(before.parse.workers); + expect(after.parse.workerHeapMb).toBe(before.parse.workerHeapMb); + expect(after.parse.reservedBytes - before.parse.reservedBytes).toBe(193 * MIB); + expect(after.embedding.reservedBytes - before.embedding.reservedBytes).toBe(193 * MIB); + }); + + it('rejects an embedding-only process family whose supervisor exceeds its remaining capacity', () => { + const fixed = { + ...request, + totalMb: 1024, + mainHeapMb: 384, + graphPoolBytes: 256 * MIB, + modelMemoryMb: 0, + includeParsing: false, + }; + expect( + resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 96 * MIB }).embedding + .reservedBytes, + ).toBe(1024 * MIB); + expect(() => resolveAnalysisResourceBudget({ ...fixed, supervisorBytes: 97 * MIB })).toThrow( + 'cannot admit', + ); + }); + + it.each([NaN, Infinity, -1, 0.5])( + 'rejects an invalid supervisor reservation %s', + (supervisorBytes) => { + expect(() => resolveAnalysisResourceBudget({ ...request, supervisorBytes })).toThrow( + 'supervisor', + ); + }, + ); + + it('bounds every phase separately and reuses parser capacity for embedding', () => { + for (const memoryMb of [2048, 4096, 8192, 16384, 32768]) { + for (const availableCpus of [1, 2, 4, 24]) { + const budget = resolveAnalysisResourceBudget({ + ...request, + effectiveMemoryBytes: memoryMb * MIB, + totalMb: Math.floor(memoryMb * 0.75), + availableCpus, + modelMemoryMb: 0, + }); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(budget.totalBytes).toBeLessThan(memoryMb * MIB); + expect(budget.parse.workers).toBeLessThanOrEqual(Math.max(1, availableCpus - 1)); + expect(budget.embedding.nativeThreads).toBeLessThanOrEqual(availableCpus); + } + } + }); + + it('preserves explicit main-heap semantics and preferred concurrency when they fit', () => { + const budget = resolveAnalysisResourceBudget({ + ...request, + totalMb: 24_576, + mainHeapMb: 8192, + preferredWorkers: 6, + }); + expect(budget.mainHeapMb).toBe(8192); + expect(budget.parse.workers).toBe(6); + expect(budget.automatic).toBe(false); + expect(Object.isFrozen(budget)).toBe(true); + expect(Object.isFrozen(budget.parse)).toBe(true); + expect(Object.isFrozen(budget.embedding)).toBe(true); + }); + + it.each([1, 4])( + 'admits an explicit worker override on %s CPU(s) within its memory budget', + (availableCpus) => { + const input = { ...request, availableCpus, mainHeapMb: 8192, workers: 12 }; + const budget = resolveAnalysisResourceBudget(input); + expect(budget.parse.workers).toBe(12); + expect(budget.parse.workerHeapMb).toBeGreaterThanOrEqual(512); + expect(budget.parse.nativeThreads).toBe(availableCpus); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => + resolveAnalysisResourceBudget({ ...input, totalMb: 24576, workerHeapMb: 4096 }), + ).toThrow('do not fit'); + }, + ); + + it('admits an HTTP embedding-only resume below the parser minimum', () => { + const input = { ...request, totalMb: 1024, modelMemoryMb: 0 }; + expect(() => resolveAnalysisResourceBudget(input)).toThrow('cannot admit'); + const budget = resolveAnalysisResourceBudget({ ...input, includeParsing: false }); + expect(budget.parse.workers).toBe(0); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(1024 * MIB); + }); + + it('preserves the heap-only flag minimum without silently raising it', () => { + const budget = resolveAnalysisResourceBudget({ ...request, mainHeapMb: 200 }); + expect(budget.mainHeapMb).toBe(200); + expect(budget.parse.nativeThreads).toBe(request.availableCpus); + expect(budget.parse.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + expect(() => resolveAnalysisResourceBudget({ ...request, mainHeapMb: 199 })).toThrow( + 'cannot admit', + ); + }); + + it('admits explicit embedding concurrency and includes sidecar heap within the model reservation', () => { + const budget = resolveAnalysisResourceBudget({ ...request, embeddingThreads: 8 }); + expect(budget.embedding.nativeThreads).toBe(8); + expect(budget.embedding.modelHeapMb).toBe(512); + expect(budget.embedding.modelHeapMb * MIB + 48 * MIB).toBeLessThan(budget.embedding.modelBytes); + expect(() => resolveAnalysisResourceBudget({ ...request, embeddingThreads: 25 })).toThrow( + 'CPU allowance', + ); + }); + + it('admits the full working set of a supported 10 MiB node with a larger budget', () => { + const budget = resolveAnalysisResourceBudget({ + ...request, + totalMb: 8192, + modelMemoryMb: 0, + includeParsing: false, + }); + expect(budget.embedding.queueBytes).toBeGreaterThan(18 * 10 * MIB + MIB); + expect(budget.embedding.reservedBytes).toBeLessThanOrEqual(budget.totalBytes); + }); + + it('rejects impossible explicit combinations before work starts', () => { + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 128 })).toThrow( + 'cannot admit', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 65536 })).toThrow( + 'exceeds effective', + ); + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 4096, mainHeapMb: 8192 }), + ).toThrow('cannot admit'); + expect(() => + resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 4, workerHeapMb: 4096 }), + ).toThrow('do not fit'); + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb: 8192, workers: 24 })).toThrow( + 'do not fit', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: 0 })).toThrow( + 'graph pool bytes', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, graphPoolBytes: MIB })).toThrow( + 'graph pool bytes', + ); + expect(() => resolveAnalysisResourceBudget({ ...request, modelMemoryMb: 128 })).toThrow( + 'model reservation', + ); + }); + + it.each([NaN, Infinity, -1, 0, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects malformed total budget %s', + (totalMb) => { + expect(() => resolveAnalysisResourceBudget({ ...request, totalMb })).toThrow(); + }, + ); +}); + +describe('embedding payload accounting', () => { + it('accounts an active payload and prefetch together, with a stable peak', () => { + const budget = new EmbeddingByteBudget(1024); + for (let i = 0; i < 100_000; i++) { + const active = budget.reserve(512); + const prefetched = budget.reserve(512); + expect(budget.usedBytes).toBe(1024); + prefetched(); + active(); + } + expect(budget.usedBytes).toBe(0); + expect(budget.peakBytes).toBe(1024); + }); + + it('rejects overflow without leaking capacity and detects double release', () => { + const budget = new EmbeddingByteBudget(1024); + const release = budget.reserve(768); + expect(() => budget.reserve(257)).toThrow('256 bytes available'); + expect(budget.usedBytes).toBe(768); + release(); + expect(() => release()).toThrow('released twice'); + expect(budget.usedBytes).toBe(0); + expect(() => budget.reserve(-1)).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/analysis-staging-retention.test.ts b/gitnexus/test/unit/analysis-staging-retention.test.ts new file mode 100644 index 000000000..56dba2060 --- /dev/null +++ b/gitnexus/test/unit/analysis-staging-retention.test.ts @@ -0,0 +1,75 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import { acquireIndexLock, sweepStagingArtifacts } from '../../src/storage/index-lock.js'; +import { + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { saveMeta } from '../../src/storage/repo-manager.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +let root: string; +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-prepared-sweep-'))); +}); +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); +}); + +it('keeps the prepared graph through ordinary lock reacquisition and then publishes it', async () => { + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'verified graph'); + const metadata: RepoMeta = { + repoPath: root, + lastCommit: 'a'.repeat(40), + indexedAt: '2026-11-04T00:00:00Z', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + }; + await saveMeta(root, metadata); + const prepared = await prepareAnalysisPublication(root, stage, metadata, 'b'.repeat(64)); + const lock = await acquireIndexLock(root); + try { + expect(await fs.readFile(stage, 'utf8')).toBe('verified graph'); + const published = await publishPreparedAnalysis(root, { + verify: async (file) => { + expect(await fs.readFile(file, 'utf8')).toBe('verified graph'); + }, + saveMetadata: (meta) => saveMeta(root, meta), + }); + expect(published?.graphGeneration?.id).toBe(prepared.id); + } finally { + lock.release(); + } +}); + +it.each(['malformed', 'directory', 'dangling-link'])( + 'retains staging evidence when the prepared receipt is %s', + async (kind) => { + const receipt = path.join(root, 'analysis-publication.json'); + if (kind === 'directory') await fs.mkdir(receipt); + else if (kind === 'dangling-link') await fs.symlink(path.join(root, 'missing'), receipt); + else await fs.writeFile(receipt, '{'); + const stage = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(stage, 'unverified but owned work'); + await fs.writeFile(stage + '.wal', 'native recovery evidence'); + sweepStagingArtifacts(root); + expect(await fs.readFile(stage, 'utf8')).toBe('unverified but owned work'); + expect(await fs.readFile(stage + '.wal', 'utf8')).toBe('native recovery evidence'); + }, +); + +it('scopes the retention marker to its own locked branch slot', async () => { + const branch = path.join(root, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await fs.writeFile(path.join(branch, 'analysis-publication.json'), '{'); + const name = `lbug.staging.${randomUUID()}`; + await fs.writeFile(path.join(branch, name), 'branch work'); + await fs.writeFile(path.join(root, name), 'flat orphan'); + sweepStagingArtifacts(root); + sweepStagingArtifacts(branch); + await expect(fs.stat(path.join(root, name))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(path.join(branch, name), 'utf8')).toBe('branch work'); +}); diff --git a/gitnexus/test/unit/analyze-api.test.ts b/gitnexus/test/unit/analyze-api.test.ts index 076e86b70..19355c978 100644 --- a/gitnexus/test/unit/analyze-api.test.ts +++ b/gitnexus/test/unit/analyze-api.test.ts @@ -591,109 +591,9 @@ describe('POST /api/embed records the embedding count it measured', () => { }); }); -/** - * Wiring guard for the route. Everything the helpers DECIDE is pinned - * behaviorally above; what remains is that the inline route closure inside - * `createServer` still asks them — the helper being right while the call site - * keeps writing `embeddingCheckpoint: undefined` is exactly the regression - * #2790 is about, and that closure cannot be reached without booting a server - * over a real repo + LadybugDB + embedding endpoint. Static-analysis layer of - * last resort, same precedent as api-readonly-wiring.test.ts. - */ -describe('POST /api/embed route wiring (#2790)', () => { - const readSource = () => - fs.readFile(path.join(__dirname, '..', '..', 'src', 'server', 'api.ts'), 'utf-8'); - - /** - * The body of the route's `withLbugDb` callback — everything that may only - * run while the database connection is open. Sliced rather than matched with - * a character-distance regex so a comment edit cannot silently un-assert it. - */ - const insideWithLbugDb = (source: string): string => { - // The open is a multi-line `withLbugDb(lbugPath, async () => {…}, opts)` call - // since #3091 added the FTS-mode options argument, so anchor on the call head - // and close on that options argument rather than a fixed-indent literal. - const head = source.match(/await withLbugDb\(\s*lbugPath,\s*async \(\) => \{/); - expect(head).not.toBeNull(); - const start = head!.index!; - const end = source.indexOf('skipFtsOption(ftsSession.skipFts)', start); - expect(end).toBeGreaterThan(start); - return source.slice(start, end); - }; - - it('feeds the pipeline result through resolveEmbedRunOutcome into the finalize write', async () => { - const source = await readSource(); - // The result is captured, not discarded… - expect(source).toContain('const pipelineResult = await runEmbeddingPipeline('); - // …handed to the helper with the finalize context… - expect(source).toMatch( - /resolveEmbedRunOutcome\(\s*embeddingIdentity,\s*pipelineResult,\s*finalizeContext,\s*\)/, - ); - // …and its checkpoint is what the finalize meta write persists (pre-fix: a - // hardcoded `embeddingCheckpoint: undefined`). - expect(source).toContain('embeddingCheckpoint: outcome.checkpoint'); - expect(source).toContain('partialRunError = outcome.error;'); - // A partial run does not reach `status: 'complete'`, and carries its detail. - expect(source).toMatch( - /partialRunError === undefined[\s\S]{0,400}status: 'complete'[\s\S]{0,600}status: 'failed'/, - ); - expect(source).toContain('partial: partialRunDetail,'); - }); - - it('measures after the WAL flush, inside withLbugDb, and folds the result into the write', async () => { - const source = await readSource(); - const region = insideWithLbugDb(source); - // Inside the open connection — this is the route's only chance to stamp - // `stats.embeddings`, and the next CLI run's preserve-or-wipe decision - // hangs on it. - expect(region).toContain('const measuredEmbeddings = await countPersistedEmbeddings();'); - expect(region).toContain('await saveMeta(storagePath, embeddingMeta);'); - // Ordering, without brittle character spans: flush → measure → decide → - // write. Counting before the flush would describe rows still in the WAL. - const flushed = region.lastIndexOf('await flushWAL();'); - const measured = region.indexOf('const measuredEmbeddings = await countPersistedEmbeddings();'); - const decided = region.indexOf('const outcome = resolveEmbedRunOutcome('); - const folded = region.indexOf('embeddingMeta = withMeasuredEmbeddingCount(', measured); - expect(flushed).toBeLessThan(measured); - expect(measured).toBeLessThan(decided); - expect(decided).toBeLessThan(folded); - expect(region.slice(folded)).toContain('measuredEmbeddings,'); - }); - - it('measures in the post-flush checkpoint callback and nowhere else in the pipeline options', async () => { - const source = await readSource(); - expect(source).toMatch( - /await saveEmbeddingCheckpoint\(\s*checkpoint,\s*\[\],\s*await countPersistedEmbeddings\(\),?\s*\)/, - ); - // The window-start callback fires before any row exists — it must pass no - // count rather than restate a stale one. - expect(source).toMatch( - /onCheckpointWindowStart: async \(\{ nodeIds, \.\.\.checkpoint \}\) => \{\s*await saveEmbeddingCheckpoint\(checkpoint, nodeIds\);\s*\},/, - ); - }); - - it('resolves a found checkpoint through the shared resume decision', async () => { - const source = await readSource(); - const region = insideWithLbugDb(source); - // The route asks the SAME decider the CLI does, instead of hard-throwing on - // any identity mismatch and ignoring `attempts` — the disagreement that let - // a CLI-written `'partial'` marker wedge every later `POST /api/embed`. - expect(region).toMatch(/decideEmbeddingResume\(priorCheckpoint, embeddingIdentity\)/); - // Every action is routed: abort fails the run, abandon warns and proceeds - // with an empty pending set, resume hands the decision's ids to the pipeline. - expect(region).toContain("if (resume?.action === 'abort') throw new Error(resume.error);"); - expect(region).toMatch(/resume\?\.action === 'resume'\s*\?\s*resume\.pendingNodeIds/); - // No second copy of the gate: the route no longer authors its own message. - expect(region).not.toContain('Cannot resume embedding checkpoint:'); - }); - - it('never maps the pipeline ready phase to a phase a client can read as terminal', async () => { - const source = await readSource(); - // `ready` fires unconditionally before the route knows the outcome (#2790). - expect(source).toMatch(/p\.phase === 'ready'\s*\?\s*'finalizing'/); - expect(source).not.toMatch(/p\.phase === 'ready' \? 'complete'/); - }); -}); +// The shared phase now owns checkpoint/count publication. Its runtime contract +// is covered by analysis-embedding-phase.test.ts and analysis-embedding-flow.test.ts; +// api-fts-mode.test.ts drives this route's progress, failure and completion wiring. describe('HTTP repo catalog validation', () => { const readSource = () => diff --git a/gitnexus/test/unit/analyze-bootstrap.test.ts b/gitnexus/test/unit/analyze-bootstrap.test.ts new file mode 100644 index 000000000..b38b7c3ce --- /dev/null +++ b/gitnexus/test/unit/analyze-bootstrap.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + capture: vi.fn(), + check: vi.fn(() => ({ ok: true })), + analyze: vi.fn(), + analyzeOrWatch: vi.fn(), +})); + +vi.mock('../../src/core/analyzer-identity.js', () => ({ + captureAnalyzerIdentityBeforeLoad: mocks.capture, +})); +vi.mock('../../src/core/lbug/native-check.js', () => ({ + prepareLbugNativeBinary: vi.fn(), + checkLbugNative: mocks.check, +})); +vi.mock('../../src/cli/analyze.js', () => ({ + analyzeCommand: mocks.analyze, + analyzeOrWatchCommandWithRunnerIdentity: mocks.analyzeOrWatch, +})); +vi.mock('../../src/cli/update-notice.js', () => ({ runProcessCliUpdateNotice: vi.fn() })); + +const originalArgv = process.argv; +const originalExecArgv = process.execArgv; +const connected = Object.getOwnPropertyDescriptor(process, 'connected'); +const receipt = { schemaVersion: 4 }; + +beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + process.execArgv = []; + for (const key of [ + 'GITNEXUS_ANALYSIS_SUPERVISED', + 'GITNEXUS_ANALYSIS_MANUAL_HEAP', + 'GITNEXUS_MEMORY', + 'GITNEXUS_TEST_RESPAWN_HEAP_MB', + 'NODE_OPTIONS', + ]) + vi.stubEnv(key, undefined); + mocks.capture.mockImplementation(async (_url, loader) => ({ + runnerIdentity: receipt, + loaded: await loader(), + })); + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + // Vitest's fork also owns IPC. Keep it connected while exercising the CLI + // dispatcher, so its own result channel is not disrupted by this fixture. + Object.defineProperty(process, 'connected', { configurable: true, value: true }); +}); + +afterEach(() => { + process.argv = originalArgv; + process.execArgv = originalExecArgv; + if (connected) Object.defineProperty(process, 'connected', connected); + else delete (process as Partial).connected; + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +it.each<{ + continuation?: boolean; + watch?: boolean; + budget?: boolean; + source?: string; + env?: Record; + execArgv?: string[]; + captures: boolean; +}>([ + { continuation: false, watch: false, budget: false, source: '', captures: false }, + { continuation: false, watch: true, budget: false, source: '', captures: true }, + { continuation: true, watch: false, budget: false, source: '', captures: false }, + { continuation: false, watch: false, budget: true, source: '', captures: false }, + { continuation: false, watch: false, budget: true, source: 'auto', captures: false }, + { continuation: false, watch: false, budget: true, source: 'budget', captures: true }, + { continuation: false, watch: true, budget: true, source: '', captures: true }, + { env: { GITNEXUS_ANALYSIS_SUPERVISED: '1' }, captures: true }, + { env: { GITNEXUS_ANALYSIS_MANUAL_HEAP: '1' }, captures: true }, + { env: { GITNEXUS_MEMORY: 'off' }, captures: true }, + { env: { GITNEXUS_TEST_RESPAWN_HEAP_MB: '128' }, captures: true }, + { env: { NODE_OPTIONS: '--max_old_space_size=2048' }, captures: true }, + { execArgv: ['--max-old-space-size=2048'], captures: true }, +])('selects the analyzer bootstrap for %j', async (scenario) => { + vi.stubEnv('GITNEXUS_ANALYSIS_CONTINUATION', scenario.continuation ? '1' : ''); + vi.stubEnv('GITNEXUS_HEAP_LIMIT_SOURCE', scenario.source ?? ''); + for (const [key, value] of Object.entries(scenario.env ?? {})) vi.stubEnv(key, value); + if (scenario.execArgv) process.execArgv = scenario.execArgv; + process.argv = [process.execPath, 'gitnexus', 'analyze', '/fixture']; + if (scenario.watch) process.argv.push('--watch'); + if (scenario.budget) process.argv.push('--memory-budget', '2048'); + + await import('../../src/cli/index.js'); + if (scenario.captures) { + await vi.waitFor(() => expect(mocks.analyzeOrWatch).toHaveBeenCalledOnce()); + expect(mocks.analyzeOrWatch).toHaveBeenCalledWith( + receipt, + '/fixture', + expect.objectContaining(scenario.watch ? { watch: true } : {}), + ); + expect(mocks.capture).toHaveBeenCalledOnce(); + expect(mocks.analyze).not.toHaveBeenCalled(); + } else { + await vi.waitFor(() => expect(mocks.analyze).toHaveBeenCalledOnce()); + expect(mocks.analyze).toHaveBeenCalledWith('/fixture', expect.any(Object)); + expect(mocks.capture).not.toHaveBeenCalled(); + expect(mocks.analyzeOrWatch).not.toHaveBeenCalled(); + } + expect(mocks.check).toHaveBeenCalledOnce(); +}); diff --git a/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts b/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts index 53ead617c..2fd7e0874 100644 --- a/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts +++ b/gitnexus/test/unit/analyze-embedding-endpoint-flags.test.ts @@ -20,7 +20,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-embeddings-limit.test.ts b/gitnexus/test/unit/analyze-embeddings-limit.test.ts index c7f12fe7f..ee4d1aa2c 100644 --- a/gitnexus/test/unit/analyze-embeddings-limit.test.ts +++ b/gitnexus/test/unit/analyze-embeddings-limit.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts b/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts index 68798111e..c8f12d425 100644 --- a/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts +++ b/gitnexus/test/unit/analyze-finalize-failure-exits.test.ts @@ -62,7 +62,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ isLbugReady: isLbugReadyMock, })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError, diff --git a/gitnexus/test/unit/analyze-gitnexusrc.test.ts b/gitnexus/test/unit/analyze-gitnexusrc.test.ts index e7fe6224c..d5604be10 100644 --- a/gitnexus/test/unit/analyze-gitnexusrc.test.ts +++ b/gitnexus/test/unit/analyze-gitnexusrc.test.ts @@ -50,9 +50,11 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: vi.fn(async () => null), getStoragePaths: vi.fn((repoPath: string) => ({ storagePath: path.join(repoPath, '.gitnexus'), lbugPath: path.join(repoPath, '.gitnexus', 'lbug'), + metaPath: path.join(repoPath, '.gitnexus', 'gitnexus.json'), })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts index 150cbb554..c1bccbf8b 100644 --- a/gitnexus/test/unit/analyze-heap-respawn.test.ts +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -3,6 +3,11 @@ import { EventEmitter } from 'node:events'; const spawnMock = vi.fn(); const getHeapStatisticsMock = vi.fn(); +const npmWarningMock = vi.hoisted(() => vi.fn()); + +vi.mock('../../src/cli/resolve-invocation.js', () => ({ + warnIfNpm11NpxRisk: npmWarningMock, +})); vi.mock('child_process', async () => { const actual = await vi.importActual('child_process'); @@ -105,7 +110,37 @@ describe('analyzeCommand heap respawn', () => { else process.env.NODE_OPTIONS = initialNodeOptions; }); - it('re-execs analyze with the auto-sized heap cap (16GB-clamped) + larger semi-space and bridges progress redraw when parent is a TTY', async () => { + it('defers npm subprocess probes until the admitted working process', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + npmWarningMock.mockClear(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(npmWarningMock).not.toHaveBeenCalled(); + }); + + it('leaves the analysis pipeline unloaded in the heap supervisor', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + const load = vi.fn(() => ({ + runFullAnalysis: vi.fn(), + continuePreparedAnalysis: vi.fn(), + })); + vi.doMock('../../src/core/run-analyze.js', load); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(load).not.toHaveBeenCalled(); + } finally { + vi.doUnmock('../../src/core/run-analyze.js'); + } + }); + + it('re-execs with the admitted main heap and bridges progress redraw when parent is a TTY', async () => { delete process.env.NODE_OPTIONS; restoreStderrIsTTY = setStreamIsTTY(process.stderr, true); getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); @@ -116,9 +151,10 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=13107'); + expect(args).toContain('--max-old-space-size=12723'); expect(args).toContain('--max-semi-space-size=128'); - expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=13107'); + expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=12723'); + expect(opts.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB).toBe('13107'); expect(opts.env.NODE_OPTIONS).toContain('--max-semi-space-size=128'); expect(opts.env.GITNEXUS_RESPAWN_PROGRESS_TTY).toBe('1'); }); @@ -161,14 +197,14 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=13107'); + expect(args).toContain('--max-old-space-size=12723'); // The auto flag is appended after the ambient value, so V8's - // later-flag-wins semantics resolve to the larger cap. - expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=13107')).toBeGreaterThan( + // later-flag-wins semantics resolve to the admitted whole-heap cap. + expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=12723')).toBeGreaterThan( opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=4096'), ); const warn = cap.records().find((r) => r.msg.includes('pins the heap to 4096MB')); - expect(warn?.msg).toContain('Re-running analyze with the larger auto-sized cap'); + expect(warn?.msg).toContain('Re-running analyze with the admitted 13107MB main heap'); }); it('honors GITNEXUS_MEMORY=off: keeps the small ambient heap, silently (#2649)', async () => { @@ -229,6 +265,47 @@ describe('analyzeCommand heap respawn', () => { expect(forwardedSignalExitCode('SIGABRT', false)).toBe(1); }); + it.each(['analysis-ready', 'analysis-watch-ready'])( + 'resends startup cancellation on %s and waits for child exit', + async (ready) => { + delete process.env.NODE_OPTIONS; + const child = Object.assign(new EventEmitter(), { + connected: true, + exitCode: null, + signalCode: null, + kill: vi.fn(), + stdout: new EventEmitter(), + stderr: new EventEmitter(), + send: vi.fn((_message, callback) => callback(null)), + }); + spawnMock.mockReturnValueOnce(child); + const { ensureHeap } = await import('../../src/cli/analyze.js'); + let settled = false; + const pending = ensureHeap({ memoryBudget: '2048' }).then(() => { + settled = true; + }); + try { + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledOnce()); + process.emit('SIGTERM'); + expect(child.send).toHaveBeenCalledExactlyOnceWith( + { type: 'cancel' }, + expect.any(Function), + ); + process.emit('SIGTERM'); + expect(process.listenerCount('SIGTERM')).toBeGreaterThan(0); + expect(child.kill).not.toHaveBeenCalled(); + child.emit('message', { type: ready }); + await Promise.resolve(); + expect(child.send).toHaveBeenCalledTimes(3); + expect(settled).toBe(false); + } finally { + child.emit('close', 0, null); + await pending; + } + expect(process.exitCode).toBe(143); + }, + ); + it('GITNEXUS_MEMORY=off also disables the default (unpinned) respawn (#2649 review)', async () => { delete process.env.NODE_OPTIONS; process.env.GITNEXUS_MEMORY = 'off'; @@ -298,9 +375,9 @@ describe('analyzeCommand heap respawn', () => { .find((r) => r.msg.includes('Analysis likely ran out of memory')); expect(oomGuidance).toBeDefined(); const msg = oomGuidance?.msg ?? ''; - expect(msg).toContain('auto-sized to 13107MB'); - expect(msg).toContain('NODE_OPTIONS="--max-old-space-size="'); - expect(msg).toContain('[your-args]'); + expect(msg).toContain('admitted main heap'); + expect(msg).toContain('--resource-budget'); + expect(msg).toContain('reduce the analysis scope'); expect(msg).toContain('native crash unrelated to heap size'); cap.restore(); }); @@ -704,7 +781,7 @@ describe('ensureHeap --memory-budget (#3137)', () => { expect({ args, source: opts.env.GITNEXUS_HEAP_LIMIT_SOURCE, records: cap.records() }).toEqual({ args: [ ...preserved, - '--max-old-space-size=13107', + '--max-old-space-size=12723', '--max-semi-space-size=128', '--stack-size=4096', ...process.argv.slice(1), diff --git a/gitnexus/test/unit/analyze-http-endpoint-error.test.ts b/gitnexus/test/unit/analyze-http-endpoint-error.test.ts index 718a9fa16..71e7836f4 100644 --- a/gitnexus/test/unit/analyze-http-endpoint-error.test.ts +++ b/gitnexus/test/unit/analyze-http-endpoint-error.test.ts @@ -58,7 +58,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-job.test.ts b/gitnexus/test/unit/analyze-job.test.ts index 382a2e078..98e74f3aa 100644 --- a/gitnexus/test/unit/analyze-job.test.ts +++ b/gitnexus/test/unit/analyze-job.test.ts @@ -1,4 +1,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; import { JobManager, isTerminalJobStatus, @@ -31,6 +36,63 @@ describe('JobManager', () => { expect(retrieved).toEqual(created); }); + it('retains terminal jobs past their TTL while the child still owns the slot', () => { + vi.useFakeTimers(); + manager.dispose(); + manager = new JobManager(); + const job = manager.createJob({ repoPath: '/tmp/repo' }); + const child = Object.assign(new EventEmitter(), { + connected: true, + exitCode: null, + signalCode: null, + send: vi.fn(), + unref: vi.fn(), + }); + manager.registerChild(job.id, child as unknown as import('node:child_process').ChildProcess); + manager.updateJob(job.id, { status: 'failed', error: 'Cancelled' }); + vi.advanceTimersByTime(70 * 60 * 1000); + expect(manager.getJob(job.id)?.status).toBe('failed'); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow( + 'Analysis already in progress', + ); + child.emit('exit', 0); + vi.advanceTimersByTime(5 * 60 * 1000); + expect(manager.getJob(job.id)).toBeUndefined(); + expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); + }); + + it('keeps the original deadline when a graph worker hands off to an embedding worker', () => { + manager.dispose(); + vi.useFakeTimers(); + manager = new JobManager(); + const job = manager.createJob({ repoPath: '/tmp/repo' }); + manager.updateJob(job.id, { status: 'analyzing' }); + const child = () => + Object.assign(new EventEmitter(), { + connected: true, + exitCode: null, + signalCode: null, + send: vi.fn(), + unref: vi.fn(), + channel: { unref: vi.fn() }, + }); + const graph = child(); + manager.registerChild(job.id, graph as any); + vi.advanceTimersByTime(20 * 60_000); + graph.emit('exit', 0); + const embeddings = child(); + manager.registerChild(job.id, embeddings as any); + vi.advanceTimersByTime(10 * 60_000 - 1); + expect(embeddings.send).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(embeddings.send).toHaveBeenCalledWith({ type: 'cancel' }, expect.any(Function)); + expect(manager.hasPendingCancel(job.id)).toBe(true); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); + embeddings.emit('exit', 0); + expect(manager.getJob(job.id)?.error).toContain('30 minute limit'); + expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); + }); + it('returns undefined for unknown job id', () => { expect(manager.getJob('nonexistent')).toBeUndefined(); }); @@ -371,12 +433,24 @@ describe('JobManager', () => { expect(manager.createJob({ repoPath: '/tmp/other' }).status).toBe('queued'); }); - it('cancelJob falls back to a signal when the IPC channel is already closed', () => { + it('finishes shutdown when a registered child fails to spawn without emitting exit', async () => { + const job = manager.createJob({ repoPath: '/tmp/repo' }); + const child = spawn(process.execPath, [], { cwd: path.join(os.tmpdir(), randomUUID()) }); + child.on('error', () => manager.releaseChild(job.id)); + const exit = vi.fn(); + child.on('exit', exit); + manager.registerChild(job.id, child); + await manager.waitForChildren(); + expect(exit).not.toHaveBeenCalled(); + expect(child.listenerCount('close')).toBe(0); + }); + + it('cancelJob leaves a disconnected native worker running until it exits', () => { const job = manager.createJob({ repoPath: '/tmp/repo' }); manager.updateJob(job.id, { status: 'analyzing' }); const signals: string[] = []; - // connected:false — requestChildShutdown skips send() and signal-kills. + // The worker's disconnect handler requests cancellation at a safe point. const fakeChild = { connected: false, exitCode: null, @@ -391,10 +465,11 @@ describe('JobManager', () => { manager.cancelJob(job.id); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('cancelJob SIGKILLs after the grace period when the worker ignores IPC', () => { + it('cancelJob stops a wedged process after grace, retaining ownership until exit', () => { manager.dispose(); vi.useFakeTimers(); manager = new JobManager(); @@ -407,6 +482,8 @@ describe('JobManager', () => { exitCode: null, signalCode: null, send: () => true, + unref: vi.fn(), + channel: { unref: vi.fn() }, kill: (signal?: string) => { signals.push(signal ?? 'SIGTERM'); return true; @@ -418,10 +495,13 @@ describe('JobManager', () => { expect(manager.cancelJob(job.id)).toBe(true); expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); + expect(signals).toEqual([]); + vi.advanceTimersByTime(45_000); expect(signals).toEqual(['SIGKILL']); + expect(() => manager.createJob({ repoPath: '/tmp/other' })).toThrow(/already in progress/); }); - it('dispose on Windows skips immediate SIGTERM and leaves the IPC grace timer', () => { + it('dispose on Windows allows cooperative shutdown before stopping a wedged process', () => { manager.dispose(); vi.useFakeTimers(); vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); @@ -446,11 +526,11 @@ describe('JobManager', () => { expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); - expect(signals).toEqual(['SIGKILL']); + expect(signals).toEqual([]); vi.restoreAllMocks(); }); - it('dispose on Unix SIGTERMs after IPC and clears the grace timer', () => { + it('dispose on Unix also waits for the cooperative native safe point', () => { manager.dispose(); vi.useFakeTimers(); vi.spyOn(process, 'platform', 'get').mockReturnValue('linux'); @@ -473,12 +553,20 @@ describe('JobManager', () => { manager.registerChild(job.id, fakeChild as any); manager.dispose(); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); vi.advanceTimersByTime(15_000); - expect(signals).toEqual(['SIGTERM']); + expect(signals).toEqual([]); vi.restoreAllMocks(); }); + it('aborts in-process embedding work before shutdown waits for it', () => { + const job = manager.createJob({ repoPath: '/tmp/embed' }); + const controller = new AbortController(); + manager.registerAbortController(job.id, controller); + manager.dispose(); + expect(controller.signal.aborted).toBe(true); + }); + it('cancelJob returns false for terminal jobs', () => { const job = manager.createJob({ repoUrl: 'https://github.com/user/repo' }); manager.updateJob(job.id, { status: 'complete' }); diff --git a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts index 02978ff4f..5bf25705f 100644 --- a/gitnexus/test/unit/analyze-launch-branch-settle.test.ts +++ b/gitnexus/test/unit/analyze-launch-branch-settle.test.ts @@ -36,6 +36,7 @@ const H = vi.hoisted(() => ({ // Set per-test: the only directory the fake filesystem reports as freshly // written. Anything else looks stale, exactly like a slot this job skipped. settledDir: '', + graphFile: 'lbug', })); const { forkMock, REPO_PATH } = H; @@ -44,6 +45,21 @@ vi.mock('child_process', async () => { return { ...actual, fork: H.forkMock }; }); +vi.mock('../../src/core/analysis-host-resources.js', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + return { + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: () => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4096 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, + }), + }; +}); + vi.mock('../../src/storage/repo-manager.js', () => ({ INDEX_METADATA_FILE: H.METADATA_FILE, })); @@ -56,6 +72,11 @@ vi.mock('../../src/storage/storage-resolver.js', () => ({ requireStoragePath: async () => H.STORAGE_PATH, })); +vi.mock('../../src/storage/shared-store.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, resolveGraphPath: (slot: string) => `${slot}/${H.graphFile}` }; +}); + vi.mock('node:fs', async () => { const actual = await vi.importActual('node:fs'); return { @@ -67,6 +88,8 @@ vi.mock('node:fs', async () => { const file = String(p); const dir = file.slice(0, Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\'))); if (H.settledDir && dir === H.settledDir) { + if (!file.endsWith(H.METADATA_FILE) && !file.endsWith(H.graphFile)) + throw Object.assign(new Error('Retired graph name'), { code: 'ENOENT' }); return { mtimeMs: Number.MAX_SAFE_INTEGER }; } return { mtimeMs: 0 }; @@ -135,6 +158,7 @@ describe('finalization gate follows the placement the run chose', () => { backendInit = vi.fn(async () => true); closeDbHandle = vi.fn(async () => {}); H.settledDir = ''; + H.graphFile = 'lbug'; }); afterEach(() => { @@ -144,6 +168,53 @@ describe('finalization gate follows the placement the run chose', () => { forkMock.mockReset(); }); + it('keeps graph readiness and embedding counters nonterminal across worker log messages', async () => { + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher()(job, REPO_PATH, {}); + const embeddingProgress = { + phase: 'embedding', + totalNodes: 3, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 1, + failedNodes: 0, + activeUnit: 0, + chunks: 2, + }; + child.emit('message', { + type: 'progress', + phase: 'embedding', + percent: 70, + message: 'Embedding', + graphPhase: 'ready', + embeddingProgress: { ...embeddingProgress, source: '/private/source', token: 1n }, + modelDownloadPercent: 100, + }); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(jobManager.getJob(job.id)?.progress).toMatchObject({ + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 100, + }); + expect(JSON.stringify(jobManager.getJob(job.id)?.progress)).not.toContain('/private'); + child.emit('message', { + type: 'progress', + phase: 'log', + percent: -1, + message: 'Native checkpoint', + }); + expect(jobManager.getJob(job.id)?.progress).toEqual({ + phase: 'log', + percent: -1, + message: 'Native checkpoint', + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 100, + }); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(backendInit).not.toHaveBeenCalled(); + }); + it('completes a branch sub-slot run, whose files are NOT in the flat slot', async () => { // Only `branches//` looks written. The flat slot is stale, so a gate // probing it would spin to the 60s timeout instead of settling here. @@ -153,7 +224,7 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, { branch: BRANCH }); child.emit('message', completeMessage(false)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(jobManager.getJob(job.id)?.error).toBeUndefined(); expect(backendInit).toHaveBeenCalledTimes(1); @@ -167,11 +238,43 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(backendInit).toHaveBeenCalledTimes(1); }); + it('settles an FTS-only repair of a published private generation', async () => { + vi.useFakeTimers(); + H.settledDir = H.STORAGE_PATH; + H.graphFile = `lbug.generation.${'a'.repeat(32)}`; + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher()(job, REPO_PATH, {}); + const message = completeMessage(true); + message.result.ftsRepairedOnly = true; + child.emit('message', message); + child.emit('exit', 0); + await vi.advanceTimersByTimeAsync(61_000); + expect(jobManager.getJob(job.id)?.status).toBe('complete'); + expect(backendInit).toHaveBeenCalledOnce(); + }); + + it('completes resumed vectors after graph visibility and actual worker exit without rewriting the graph', async () => { + vi.useFakeTimers(); + const releaseRepoLock = vi.fn(); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher({ releaseRepoLock })(job, REPO_PATH, { embeddings: true }); + child.emit('message', { type: 'graph-ready' }); + await vi.advanceTimersByTimeAsync(0); + expect(child.send).toHaveBeenCalledWith({ type: 'graph-visible' }); + child.emit('message', completeMessage(true)); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(releaseRepoLock).not.toHaveBeenCalled(); + child.emit('exit', 0); + await vi.advanceTimersByTimeAsync(61_000); + expect(jobManager.getJob(job.id)?.status).toBe('complete'); + expect(releaseRepoLock).toHaveBeenCalledOnce(); + }); + it('settles a first-pin (branch SET, isPrimaryBranch true) against the flat slot', async () => { // Fresh clone: first pin adopts the flat slot. The slug dir is stale, so a // gate that does `branch ? slugDir : flat` would spin the 60s timeout here. @@ -181,7 +284,7 @@ describe('finalization gate follows the placement the run chose', () => { await launcher()(job, REPO_PATH, { branch: BRANCH }); child.emit('message', completeMessage(true)); - + child.emit('exit', 0); await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); expect(jobManager.getJob(job.id)?.error).toBeUndefined(); expect(backendInit).toHaveBeenCalledTimes(1); @@ -243,6 +346,7 @@ describe('finalization gate follows the placement the run chose', () => { expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); // Must match FINALIZE_SETTLE_TIMEOUT_MS + one poll in analyze-launch.ts. + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(61_000); const done = jobManager.getJob(job.id); @@ -261,6 +365,7 @@ describe('finalization gate follows the placement the run chose', () => { const job = jobManager.createJob({ repoPath: REPO_PATH }); await launcher({ releaseRepoLock })(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); jobManager.cancelJob(job.id, 'Cancelled by user'); @@ -274,9 +379,6 @@ describe('finalization gate follows the placement the run chose', () => { expect(done?.error).toBe('Cancelled by user'); expect(done?.error).not.toMatch(/finalization not visible/); expect(backendInit).not.toHaveBeenCalled(); - expect(releaseRepoLock).not.toHaveBeenCalled(); - - child.emit('exit', 0); expect(releaseRepoLock).toHaveBeenCalledTimes(1); }); @@ -288,10 +390,10 @@ describe('finalization gate follows the placement the run chose', () => { const job = jobManager.createJob({ repoPath: REPO_PATH }); await launcher({ releaseRepoLock })(job, REPO_PATH, {}); child.emit('message', completeMessage(true)); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); jobManager.cancelJob(job.id, 'Cancelled by user'); - child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('failed'); expect(releaseRepoLock).not.toHaveBeenCalled(); @@ -326,6 +428,7 @@ describe('finalization gate follows the placement the run chose', () => { expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); H.settledDir = H.STORAGE_PATH; + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(200); expect(jobManager.getJob(job.id)?.status).toBe('complete'); @@ -334,6 +437,27 @@ describe('finalization gate follows the placement the run chose', () => { expect(order).toEqual(['backend.init', 'releaseRepoLock']); }); + it.each(['before-exit', 'during-retry'] as const)( + 'does not restart a worker when shutdown begins %s', + async (when) => { + vi.useFakeTimers(); + const releaseRepoLock = vi.fn(); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launcher({ releaseRepoLock })(job, REPO_PATH, {}); + + if (when === 'before-exit') jobManager.dispose(); + child.emit('exit', 1); + if (when === 'during-retry') jobManager.dispose(); + await jobManager.waitForChildren(); + await vi.advanceTimersByTimeAsync(5_000); + + expect(forkMock).toHaveBeenCalledTimes(1); + expect(jobManager.getJob(job.id)?.status).toBe('failed'); + expect(jobManager.getJob(job.id)?.error).toBe('Server shutting down'); + expect(releaseRepoLock).toHaveBeenCalledTimes(1); + }, + ); + it('still treats an exit with no terminal IPC as a crash worth retrying', async () => { // The guard must not swallow real crashes: no `complete`/`error` was sent. H.settledDir = H.STORAGE_PATH; diff --git a/gitnexus/test/unit/analyze-launch-collapse.test.ts b/gitnexus/test/unit/analyze-launch-collapse.test.ts index 4172ead9a..1b9a32f89 100644 --- a/gitnexus/test/unit/analyze-launch-collapse.test.ts +++ b/gitnexus/test/unit/analyze-launch-collapse.test.ts @@ -19,6 +19,9 @@ */ import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; import { EventEmitter } from 'node:events'; +import * as hostResources from '../../src/core/analysis-host-resources.js'; +import * as effectiveRam from '../../src/core/ingestion/utils/effective-ram.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; // `vi.mock` factories are hoisted above every top-level `const`, and this file // imports the module under test statically — so anything a factory closes over @@ -46,6 +49,21 @@ vi.mock('child_process', async () => { // ownership-validated storage path. Pin the filesystem so the gate settles on // its FIRST poll — the gate itself is not under test here and its 200ms poll // would otherwise put a real timer between the worker message and the assertions. +vi.mock('../../src/core/analysis-host-resources.js', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + return { + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: () => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 4096 * 1024 ** 2, + availableCpus: 2, + modelMemoryMb: 0, + }), + }; +}); + vi.mock('../../src/storage/repo-manager.js', () => ({ INDEX_METADATA_FILE: H.METADATA_FILE, })); @@ -135,6 +153,9 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () const job = jobManager.createJob({ repoPath: REPO_PATH }); await launch(job, REPO_PATH, {}); child.emit('message', msg); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + expect(backendInit).not.toHaveBeenCalled(); + child.emit('exit', 0); await vi.waitFor(() => expect(calls).toContain('updateJob:terminal')); return jobManager.getJob(job.id); @@ -191,6 +212,93 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () ); }); + it.each([ + [undefined, 8192], + ['4096', 4096], + ] as const)('preserves the server heap cap with override %s', async (override, heap) => { + vi.stubEnv('GITNEXUS_SERVER_ANALYZE_HEAP_MB', override); + const cap = vi.spyOn(effectiveRam, 'autoHeapCapMb').mockReturnValue(24576); + const admission = vi + .spyOn(hostResources, 'resolveHostAnalysisResources') + .mockImplementation((options) => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 32 * 1024 ** 3, + availableCpus: 8, + mainHeapMb: options.mainHeapMb, + modelMemoryMb: 0, + }), + ); + try { + expect((await runWorker(completeMessage()))?.status).toBe('complete'); + expect(admission).toHaveBeenCalledWith(expect.objectContaining({ mainHeapMb: heap + 48 })); + expect(forkMock).toHaveBeenCalledWith( + expect.any(String), + [], + expect.objectContaining({ + execArgv: expect.arrayContaining([`--max-old-space-size=${heap}`]), + }), + ); + } finally { + cap.mockRestore(); + admission.mockRestore(); + vi.unstubAllEnvs(); + } + }); + + it('publishes the graph before acknowledging paid work and waits for both exits', async () => { + child.pid = 12345; + const writer = child; + const embedding = makeChild(); + forkMock.mockReturnValueOnce(writer).mockReturnValueOnce(embedding); + let visible!: () => void; + backendInit.mockImplementationOnce( + () => + new Promise((resolve) => { + visible = resolve; + }), + ); + const releaseRepoLock = vi.fn(); + const launch = createLaunchAnalysisWorker({ + jobManager, + backend: { init: backendInit }, + acquireRepoLock: () => null, + releaseRepoLock, + closeDbHandle, + }); + const job = jobManager.createJob({ repoPath: REPO_PATH }); + await launch(job, REPO_PATH, { embeddings: true }); + const handoff = { + version: 1, + publicationId: 'a'.repeat(32), + repoPath: REPO_PATH, + storagePath: H.STORAGE_PATH, + metadataDirectory: H.STORAGE_PATH, + repoName: REPO_NAME, + commit: 'b'.repeat(40), + }; + writer.emit('message', { type: 'graph-prepared', handoff, result: completeMessage().result }); + expect(forkMock).toHaveBeenCalledTimes(1); + expect(backendInit).not.toHaveBeenCalled(); + writer.emit('exit', 0); + expect(forkMock).toHaveBeenCalledTimes(2); + expect(embedding.send).toHaveBeenCalledWith( + expect.objectContaining({ type: 'continue', writerPid: writer.pid }), + ); + embedding.emit('message', { type: 'graph-ready' }); + await vi.waitFor(() => expect(backendInit).toHaveBeenCalledOnce()); + expect(embedding.send).not.toHaveBeenCalledWith({ type: 'graph-visible' }); + expect(releaseRepoLock).not.toHaveBeenCalled(); + visible(); + await vi.waitFor(() => expect(embedding.send).toHaveBeenCalledWith({ type: 'graph-visible' })); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + embedding.emit('message', completeMessage()); + expect(releaseRepoLock).not.toHaveBeenCalled(); + expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); + embedding.emit('exit', 0); + await vi.waitFor(() => expect(jobManager.getJob(job.id)?.status).toBe('complete')); + expect(releaseRepoLock).toHaveBeenCalledOnce(); + }); + describe('route configuration', () => { const launchWith = async () => { const launch = createLaunchAnalysisWorker({ @@ -444,6 +552,7 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); // Must match FINALIZE_SETTLE_TIMEOUT_MS + one poll in analyze-launch.ts. + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(61_000); const done = jobManager.getJob(job.id); @@ -477,6 +586,7 @@ describe('createLaunchAnalysisWorker — collapsed index is never published', () expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); H.settleOk = true; + child.emit('exit', 0); await vi.advanceTimersByTimeAsync(200); expect(jobManager.getJob(job.id)?.status).toBe('complete'); @@ -548,6 +658,7 @@ describe('createLaunchAnalysisWorker — pending cancel', () => { H.settleOk = false; const job = await launchOne(); child.emit('message', completeMessage()); + child.emit('exit', 0); expect(jobManager.getJob(job.id)?.status).toBe('analyzing'); expect(jobManager.cancelJob(job.id, 'Cancelled by user')).toBe(true); H.settleOk = true; diff --git a/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts b/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts index 5e7f5190c..6ded0c2f4 100644 --- a/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts +++ b/gitnexus/test/unit/analyze-lbug-checkpoint-threshold.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-local-embedding-error.test.ts b/gitnexus/test/unit/analyze-local-embedding-error.test.ts index 970dfd446..6aa4d30d8 100644 --- a/gitnexus/test/unit/analyze-local-embedding-error.test.ts +++ b/gitnexus/test/unit/analyze-local-embedding-error.test.ts @@ -66,7 +66,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts index 63a417b2e..347e9f351 100644 --- a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts +++ b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts @@ -42,7 +42,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-pagesize-error.test.ts b/gitnexus/test/unit/analyze-pagesize-error.test.ts index 687816f21..3e529450b 100644 --- a/gitnexus/test/unit/analyze-pagesize-error.test.ts +++ b/gitnexus/test/unit/analyze-pagesize-error.test.ts @@ -29,7 +29,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-phase-handoff.test.ts b/gitnexus/test/unit/analyze-phase-handoff.test.ts new file mode 100644 index 000000000..fdc3263d6 --- /dev/null +++ b/gitnexus/test/unit/analyze-phase-handoff.test.ts @@ -0,0 +1,368 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; +import type { AnalyzeResult } from '../../src/core/run-analyze.js'; +import type { CliPreparedAnalysisMessage } from '../../src/cli/analyze-phase-ipc.js'; + +type SendCallback = (error: Error | null) => void; + +const mocks = vi.hoisted(() => ({ + run: vi.fn(), + spawn: vi.fn(), + skills: vi.fn(async () => ({ + skills: [{ name: 'community' }], + outputPath: '/repo/.claude/skills', + })), + context: vi.fn(async () => ({ files: [] })), + finalized: vi.fn(async () => undefined), + commit: vi.fn(), + snapshot: vi.fn( + () => + new Map([ + ['AGENTS.md', false], + ['CLAUDE.md', true], + ]), + ), + nativeReady: vi.fn(() => false), + channel: { + connected: true, + send: vi.fn((_message: unknown, callback: SendCallback) => callback(null)), + }, +})); +vi.mock('child_process', async (importOriginal) => ({ + ...(await importOriginal()), + spawn: mocks.spawn, +})); +vi.mock('../../src/cli/analyze-phase-ipc.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requireCliAnalysisSupervisor: () => actual.requireCliAnalysisSupervisor(mocks.channel), + sendCliPreparedAnalysis: (message: CliPreparedAnalysisMessage) => + actual.sendCliPreparedAnalysis(message, mocks.channel), + }; +}); +vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: mocks.run })); +vi.mock('../../src/cli/ai-context.js', () => ({ generateAIContextFiles: mocks.context })); +vi.mock('../../src/cli/skill-gen.js', () => ({ generateSkillFiles: mocks.skills })); +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + closeLbug: vi.fn(), + closeLbugBeforeExit: vi.fn(), + isLbugReady: mocks.nativeReady, + LbugWipeError: class LbugWipeError extends Error {}, +})); +vi.mock('../../src/storage/repo-manager.js', () => ({ + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '/repo/.gitnexus', + metaPath: '/repo/.gitnexus/gitnexus.json', + })), + getGlobalRegistryPath: vi.fn(() => '/nonexistent/registry.json'), + RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, + AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, + assertAnalysisFinalized: mocks.finalized, +})); +vi.mock('../../src/storage/git.js', () => ({ + getGitRoot: vi.fn(() => '/repo'), + hasGitDir: vi.fn(() => true), + getDefaultBranch: vi.fn(() => null), + selfCommitContextFiles: mocks.commit, + snapshotSelfCommitSafety: mocks.snapshot, +})); +vi.mock('../../src/core/ingestion/utils/max-file-size.js', () => ({ + getMaxFileSizeBannerMessage: vi.fn(() => null), +})); + +const send = mocks.channel.send; +const baselineUnhandled = process.listeners('unhandledRejection'); +const baselineUncaught = process.listeners('uncaughtException'); +let result: AnalyzeResult; + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv('NODE_OPTIONS', '--max-old-space-size=8192'); + process.exitCode = undefined; + mocks.nativeReady.mockReturnValue(false); + result = { + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + repoName: 'registered-repo', + stats: { files: 1, nodes: 10, edges: 20, communities: 1, processes: 2 }, + rebuildReasons: [], + preparedGraph: { + version: 1, + publicationId: 'a'.repeat(32), + commit: 'b'.repeat(40), + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + metadataDirectory: '/repo/.gitnexus', + repoName: 'registered-repo', + }, + pipelineResult: { + graph: { + toJSON: () => { + throw new Error('Live graph serialized'); + }, + }, + }, + }; + mocks.run.mockResolvedValue(result); + send.mockImplementation((message, callback) => { + JSON.stringify(message); + callback(null); + }); + mocks.channel.connected = true; + vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + vi.spyOn(console, 'log').mockImplementation(() => undefined); +}); + +afterEach(() => { + for (const listener of process.listeners('unhandledRejection')) { + if (!baselineUnhandled.includes(listener)) + process.removeListener('unhandledRejection', listener); + } + for (const listener of process.listeners('uncaughtException')) { + if (!baselineUncaught.includes(listener)) process.removeListener('uncaughtException', listener); + } + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + process.exitCode = undefined; +}); + +describe('CLI graph phase handoff', () => { + it('announces cancellation readiness before starting graph work', async () => { + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISED', '1'); + vi.stubEnv('GITNEXUS_ANALYSIS_MANUAL_HEAP', '1'); + const sendReady = vi.spyOn(process, 'send').mockImplementation((message, callback) => { + if ((message as { type?: string }).type === 'analysis-ready') { + process.emit('message', { type: 'cancel' }, undefined); + if (typeof callback === 'function') callback(null); + } + return true; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(sendReady).toHaveBeenCalledWith({ type: 'analysis-ready' }, expect.any(Function)); + expect(mocks.run).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(130); + }); + + it.each(['SIGINT', 'SIGTERM', 'ipc'] as const)( + 'waits for active work after %s and removes cancellation listeners', + async (kind) => { + const events = ['SIGINT', 'SIGTERM', 'message', 'disconnect'] as const; + const before = events.map((event) => process.listenerCount(event)); + let finish!: (value: AnalyzeResult) => void; + mocks.run.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const pending = analyzeCommand(undefined, {}); + await vi.waitFor(() => expect(mocks.run).toHaveBeenCalledOnce()); + if (kind === 'ipc') process.emit('message', { type: 'cancel' }, undefined); + else process.emit(kind); + expect(mocks.run.mock.calls[0][2].signal.aborted).toBe(true); + expect(process.exit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + finish(result); + await pending; + expect(process.exitCode).toBe(kind === 'SIGTERM' ? 143 : 130); + expect(send).not.toHaveBeenCalled(); + expect(events.map((event) => process.listenerCount(event))).toEqual(before); + }, + ); + + it.each(['pin', 'off', 'ambient'] as const)( + 'keeps %s heap flags when graph work requires a supervisor', + async (kind) => { + vi.stubEnv('GITNEXUS_MEMORY', kind === 'off' ? 'off' : ''); + vi.stubEnv( + 'NODE_OPTIONS', + kind === 'ambient' ? '--max_old_space_size=65536 --max-semi-space-size=16' : '', + ); + const descriptor = Object.getOwnPropertyDescriptor(process, 'execArgv')!; + const flags = + kind === 'pin' ? ['--max_old_space_size', '2048', '--max-semi-space-size=16'] : []; + Object.defineProperty(process, 'execArgv', { configurable: true, value: flags }); + try { + const { AnalysisSupervisorRequired } = + await import('../../src/core/analysis-supervision.js'); + mocks.run.mockRejectedValueOnce(new AnalysisSupervisorRequired()); + mocks.spawn.mockImplementationOnce(() => { + const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + }); + queueMicrotask(() => child.emit('close', 0, null)); + return child; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(mocks.spawn).toHaveBeenCalledOnce(); + const [, args, options] = mocks.spawn.mock.calls[0]; + expect(args).toEqual([...flags, '--stack-size=4096', ...process.argv.slice(1)]); + expect(options.env.NODE_OPTIONS).toBe(process.env.NODE_OPTIONS); + expect(options.env.GITNEXUS_ANALYSIS_MANUAL_HEAP).toBe('1'); + } finally { + Object.defineProperty(process, 'execArgv', descriptor); + } + }, + ); + + it('finishes live-graph skills before IPC and defers finalization, self-commit and success', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const output = console.log; + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + + expect(send).toHaveBeenCalledTimes(1); + expect(mocks.snapshot.mock.invocationCallOrder[0]).toBeLessThan( + mocks.run.mock.invocationCallOrder[0], + ); + expect(mocks.skills).toHaveBeenCalledWith('/repo', 'registered-repo', result.pipelineResult); + expect(mocks.context.mock.invocationCallOrder[0]).toBeLessThan( + send.mock.invocationCallOrder[0], + ); + expect(send.mock.calls[0][0]).toMatchObject({ + type: 'graph-prepared', + handoff: result.preparedGraph, + selfCommitSafety: { agentsMd: false, claudeMd: true }, + result: { repoName: 'registered-repo', stats: result.stats }, + }); + expect(JSON.stringify(send.mock.calls[0][0]).length).toBeLessThan(2048); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(output).not.toHaveBeenCalledWith(expect.stringContaining('indexed successfully')); + expect(process.exitCode).toBe(0); + }); + + it('keeps --index-only skill suppression and does not snapshot self-commit when omitted', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, indexOnly: true }); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.context).not.toHaveBeenCalled(); + expect(mocks.snapshot).not.toHaveBeenCalled(); + expect(send.mock.calls[0][0]).not.toHaveProperty('selfCommitSafety', expect.anything()); + expect(process.exitCode).toBe(0); + }); + + it('waits for IPC delivery before the native-handle exit, including channel backpressure', async () => { + let delivered!: SendCallback; + let received!: () => void; + const receiving = new Promise((resolve) => { + received = resolve; + }); + send.mockImplementationOnce((_message, callback) => { + delivered = callback; + received(); + return false; + }); + mocks.nativeReady.mockReturnValue(true); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const running = analyzeCommand(undefined, {}); + await receiving; + expect(process.exit).not.toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + delivered(null); + await running; + expect(process.exit).toHaveBeenCalledExactlyOnceWith(0); + }); + + it.each(['throw', 'callback'] as const)( + 'reports an IPC %s failure without success or self-commit', + async (mode) => { + send.mockImplementationOnce((_message, callback) => { + const error = new Error('Supervisor disconnected'); + if (mode === 'throw') throw error; + callback(error); + }); + mocks.nativeReady.mockReturnValue(true); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + const output = console.log; + await analyzeCommand(undefined, { selfCommit: true }); + expect(process.exit).toHaveBeenCalledExactlyOnceWith(1); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(output).not.toHaveBeenCalledWith(expect.stringContaining('indexed successfully')); + }, + ); + + it('fails before skill writes when the supervisor is missing', async () => { + mocks.channel.connected = false; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('fails if the supervisor disconnects during child-local postprocessing', async () => { + mocks.context.mockImplementationOnce(async () => { + mocks.channel.connected = false; + return { files: [] }; + }); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).toHaveBeenCalledOnce(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it.each([ + { repoName: 'different-registration' }, + { repoPath: '/other' }, + { storagePath: '/other' }, + { alreadyUpToDate: true }, + { ftsRepairedOnly: true }, + { graphWriteCollapsed: { expected: 20, persisted: 0 } }, + ])('rejects an incompatible result before skills: %j', async (invalid) => { + Object.assign(result, invalid); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true, selfCommit: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(mocks.finalized).not.toHaveBeenCalled(); + expect(mocks.commit).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('projects only named counters and safety flags, without traversing future payload fields', async () => { + const poison = { + enumerable: true, + get() { + throw new Error('Unbounded field traversed'); + }, + }; + Object.defineProperty(result.stats, 'perFile', poison); + Object.defineProperty(result.preparedGraph, 'sourceMaps', poison); + Object.defineProperty(result, 'sourceMaps', poison); + mocks.snapshot.mockReturnValueOnce( + new Map([ + ['unrelated-file', true], + ['CLAUDE.md', true], + ]), + ); + result.ftsSkipped = true; + result.ftsSkipReason = 'disabled-by-flag'; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { selfCommit: true }); + expect(send.mock.calls[0][0]).toMatchObject({ + selfCommitSafety: { agentsMd: false, claudeMd: true }, + result: { ftsSkipped: true, ftsSkipReason: 'disabled-by-flag' }, + }); + expect(JSON.stringify(send.mock.calls[0][0])).not.toMatch(/perFile|sourceMaps|unrelated-file/); + expect(process.exitCode).toBe(0); + }); + + it.each([NaN, Infinity, -1, 1.5])('rejects invalid summary counter %s', async (nodes) => { + result.stats.nodes = nodes; + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { skills: true }); + expect(process.exitCode).toBe(1); + expect(mocks.skills).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/analyze-process-budget.test.ts b/gitnexus/test/unit/analyze-process-budget.test.ts index c186ab3f4..3abc6bb07 100644 --- a/gitnexus/test/unit/analyze-process-budget.test.ts +++ b/gitnexus/test/unit/analyze-process-budget.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts b/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts index a5bfb16ee..0a861d1d8 100644 --- a/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts +++ b/gitnexus/test/unit/analyze-respawn-progress-terminal.test.ts @@ -45,7 +45,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-self-commit-bridge.test.ts b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts index d8c69dcb9..41de56c6d 100644 --- a/gitnexus/test/unit/analyze-self-commit-bridge.test.ts +++ b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts @@ -56,7 +56,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts b/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts index 99aa17547..2a3a592f9 100644 --- a/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts +++ b/gitnexus/test/unit/analyze-undeclared-pair-error.test.ts @@ -38,7 +38,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-wal-error.test.ts b/gitnexus/test/unit/analyze-wal-error.test.ts index 097175aa2..7a73fbe6c 100644 --- a/gitnexus/test/unit/analyze-wal-error.test.ts +++ b/gitnexus/test/unit/analyze-wal-error.test.ts @@ -30,7 +30,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-wipe-error.test.ts b/gitnexus/test/unit/analyze-wipe-error.test.ts index d6d881ffa..7f405d120 100644 --- a/gitnexus/test/unit/analyze-wipe-error.test.ts +++ b/gitnexus/test/unit/analyze-wipe-error.test.ts @@ -48,7 +48,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index 2c0323220..fb64ebb2b 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -20,6 +20,7 @@ import { import type { AnalyzeResult } from '../../src/core/run-analyze.js'; import type { WorkerMessage } from '../../src/server/analyze-worker.js'; import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; +import type { AnalysisPhaseProgress } from '../../src/core/analysis-progress.js'; import { IndexLockTimeoutError, type LockRecord } from '../../src/storage/index-lock.js'; const baseResult: AnalyzeResult = { @@ -36,6 +37,143 @@ const okRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => baseResul const okFinalize: WorkerAnalysisDeps['assertAnalysisFinalized'] = vi.fn(async () => undefined); const alwaysClaim: WorkerAnalysisDeps['claimTerminal'] = () => true; +it('carries bounded phase progress over worker IPC without serializing internal fields', async () => { + const send = vi.fn<(msg: WorkerMessage) => void>((message) => { + JSON.stringify(message); + }); + const embeddingProgress = { + phase: 'embedding' as const, + totalNodes: 3, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 1, + failedNodes: 0, + activeUnit: 0, + chunks: 2, + }; + const details = { + graphPhase: 'ready', + embeddingProgress: { ...embeddingProgress, identity: 1n }, + modelDownloadPercent: 25, + sourcePath: '/private/source', + native: 1n, + } as unknown as AnalysisPhaseProgress; + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async (_repo, _options, callbacks) => { + callbacks.onProgress('embedding', 70, 'Embedding current selection', details); + return baseResult; + }), + assertAnalysisFinalized: okFinalize, + claimTerminal: alwaysClaim, + send, + }, + ); + expect( + send.mock.calls.filter(([message]) => message.type === 'progress').map(([message]) => message), + ).toEqual([ + { + type: 'progress', + phase: 'embedding', + percent: 70, + message: 'Embedding current selection', + graphPhase: 'ready', + embeddingProgress, + modelDownloadPercent: 25, + }, + ]); +}); + +describe('prepared graph worker handoff', () => { + const handoff = { + version: 1 as const, + publicationId: 'a'.repeat(32), + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + metadataDirectory: '/repo/.gitnexus', + repoName: 'repo', + commit: 'b'.repeat(40), + }; + + it('sends a nonterminal bounded receipt without traversing the graph or finalizing early', async () => { + const claimTerminal = vi.fn(createTerminalClaim()); + const send = vi.fn(); + const finalize = vi.fn(async () => undefined); + const run = vi.fn(async () => ({ + ...baseResult, + preparedGraph: handoff, + get pipelineResult(): never { + throw new Error('live graph traversed'); + }, + })); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: run, + assertAnalysisFinalized: finalize, + claimTerminal, + isCancelled: () => false, + send, + }, + ); + expect(finalize).not.toHaveBeenCalled(); + expect(claimTerminal).not.toHaveBeenCalled(); + expect(claimTerminal()).toBe(true); // A later cancellation still owns one terminal outcome. + expect(send).toHaveBeenCalledOnce(); + const message = send.mock.calls[0][0]; + expect(message.type).toBe('graph-prepared'); + expect(message.handoff).toEqual(handoff); + expect(message.result.preparedGraph).toBeUndefined(); + expect(JSON.stringify(message).length).toBeLessThan(2048); + expect(run.mock.calls[0][1].skipNativeCloseOnExit).toBe(true); + }); + + it('suppresses a prepared receipt after cancellation', async () => { + const send = vi.fn(); + const claimTerminal = vi.fn(); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async () => ({ ...baseResult, preparedGraph: handoff })), + assertAnalysisFinalized: okFinalize, + claimTerminal, + isCancelled: () => true, + send, + }, + ); + expect(send).not.toHaveBeenCalled(); + expect(claimTerminal).not.toHaveBeenCalled(); + }); + + it('refuses a collapsed graph before sending its prepared receipt', async () => { + const send = vi.fn(); + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: vi.fn(async () => ({ + ...baseResult, + preparedGraph: handoff, + graphWriteCollapsed: { expected: 100, persisted: 1 }, + })), + assertAnalysisFinalized: okFinalize, + claimTerminal: createTerminalClaim(), + isCancelled: () => false, + send, + }, + ); + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith({ + type: 'error', + message: 'Collapsed graph cannot be prepared for publication', + }); + }); +}); + describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { it('reports error (not complete) when finalization fails for an unregistered repo', async () => { const send = vi.fn<(msg: WorkerMessage) => void>(); diff --git a/gitnexus/test/unit/analyze-worker-pool-size.test.ts b/gitnexus/test/unit/analyze-worker-pool-size.test.ts index dcfc17fb6..5d5b08f23 100644 --- a/gitnexus/test/unit/analyze-worker-pool-size.test.ts +++ b/gitnexus/test/unit/analyze-worker-pool-size.test.ts @@ -2,6 +2,19 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const runFullAnalysisMock = vi.fn(); +vi.mock('node:os', async (importOriginal) => { + const actual = await importOriginal(); + const mocked = { ...actual, availableParallelism: () => 4 }; + return { ...mocked, default: mocked }; +}); + +// This suite checks option forwarding, not the host's capacity to run 12 +// workers. Resource admission has its own constrained-host tests. +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), + effectiveRamBytes: () => 32 * 1024 ** 3, +})); + vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: runFullAnalysisMock, })); @@ -13,7 +26,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/analyze-worker-timeout.test.ts b/gitnexus/test/unit/analyze-worker-timeout.test.ts index 8a2dc04ef..5b1c582c5 100644 --- a/gitnexus/test/unit/analyze-worker-timeout.test.ts +++ b/gitnexus/test/unit/analyze-worker-timeout.test.ts @@ -13,7 +13,12 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ })); vi.mock('../../src/storage/repo-manager.js', () => ({ - getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + loadMeta: vi.fn(async () => null), + getStoragePaths: vi.fn(() => ({ + storagePath: '.gitnexus', + lbugPath: '.gitnexus/lbug', + metaPath: '.gitnexus/gitnexus.json', + })), getGlobalRegistryPath: vi.fn(() => 'registry.json'), RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index c7a986e7f..823f0978c 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -3,8 +3,21 @@ import { EventEmitter } from 'node:events'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { _captureLogger } from '../../src/core/logger.js'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), + effectiveRamBytes: () => 8 * 1024 ** 3, +})); +vi.mock('node:v8', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getHeapStatistics: () => ({ ...actual.getHeapStatistics(), heap_size_limit: 64 * 1024 ** 3 }), + }; +}); + const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), saveMeta: vi.fn(), @@ -13,9 +26,20 @@ const mocks = vi.hoisted(() => ({ releaseIndexLock: vi.fn(), ensurePrivateSharedGraph: vi.fn(), runEmbeddingPipeline: vi.fn(), + semanticSearch: vi.fn(async () => [ + { nodeId: 'Function:a.ts:alpha', name: 'alpha', distance: 0 }, + ]), + embedQuery: vi.fn(async () => [1, 0, 0, 0]), + isEmbedderReady: vi.fn(() => false), + runAnalysisEmbeddingPhase: vi.fn(), withLbugDb: vi.fn(), search: vi.fn(), updateJob: vi.fn(), + disconnect: vi.fn(async () => {}), + serverClose: vi.fn(), + activeJob: undefined as + | { id: string; repoName: string; status: string; progress: { graphPhase?: 'ready' } } + | undefined, })); vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ @@ -33,6 +57,14 @@ vi.mock('../../src/core/shared-store-analyze.js', () => ({ })); vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline: mocks.runEmbeddingPipeline, + semanticSearch: mocks.semanticSearch, +})); +vi.mock('../../src/core/embeddings/embedder.js', () => ({ + isEmbedderReady: mocks.isEmbedderReady, +})); +vi.mock('../../src/mcp/core/embedder.js', () => ({ embedQuery: mocks.embedQuery })); +vi.mock('../../src/core/analysis-embedding-phase.js', () => ({ + runAnalysisEmbeddingPhase: mocks.runAnalysisEmbeddingPhase, })); vi.mock('../../src/storage/storage-resolver.js', async (importOriginal) => ({ ...(await importOriginal()), @@ -55,6 +87,7 @@ vi.mock('../../src/mcp/local/local-backend.js', () => ({ async init() { return true; } + disconnect = mocks.disconnect; }, })); vi.mock('../../src/server/mcp-http.js', () => ({ @@ -72,18 +105,25 @@ vi.mock('../../src/server/grep-scan.js', () => ({ })); vi.mock('../../src/server/sse-progress.js', () => ({ mountSSEProgress: vi.fn() })); vi.mock('../../src/server/analyze-job.js', () => ({ - isTerminalJobStatus: vi.fn(() => true), + isTerminalJobStatus: (status: string) => status === 'complete' || status === 'failed', JobManager: class { + controllers: AbortController[] = []; createJob() { return { id: 'embed-job', status: 'queued' }; } updateJob = mocks.updateJob; - registerAbortController() {} + registerAbortController(_id: string, controller: AbortController) { + this.controllers.push(controller); + } + dispose() { + for (const controller of this.controllers) controller.abort(); + } + async waitForChildren() {} getJob() { - return { status: 'complete' }; + return mocks.activeJob ?? { status: 'analyzing' }; } listJobs() { - return []; + return mocks.activeJob ? [mocks.activeJob] : []; } }, })); @@ -111,7 +151,7 @@ beforeAll(async () => { ) { app = this; queueMicrotask(args.at(-1)); - return new EventEmitter() as any; + return Object.assign(new EventEmitter(), { close: mocks.serverClose }) as any; }); try { await createServer(0); @@ -133,6 +173,8 @@ afterAll(() => { beforeEach(() => { vi.clearAllMocks(); + mocks.isEmbedderReady.mockReturnValue(false); + mocks.activeJob = undefined; mocks.acquireIndexLock.mockResolvedValue({ release: mocks.releaseIndexLock, record: { @@ -146,6 +188,7 @@ beforeEach(() => { }, }); mocks.ensurePrivateSharedGraph.mockResolvedValue(true); + mocks.runAnalysisEmbeddingPhase.mockResolvedValue({ stats: { embeddings: 2 } }); mocks.listRegisteredRepos.mockResolvedValue([entry]); mocks.withLbugDb.mockImplementation(async (_path, callback) => callback()); mocks.search.mockImplementation(async (_query, _limit, _exec, reason) => ({ @@ -154,8 +197,39 @@ beforeEach(() => { })); }); +it.each(['analyzing', 'failed'])( + 'releases a waiting graph query after publication while embedding is %s', + async (status) => { + vi.useFakeTimers(); + mocks.activeJob = { + id: 'analysis-job', + repoName: entry.name, + status: 'analyzing', + progress: {}, + }; + mocks.listRegisteredRepos.mockResolvedValueOnce([]).mockResolvedValue([entry]); + mocks.loadMeta.mockResolvedValue({ capabilities: { fts: { status: 'available' } } }); + const pending = invoke('/api/search', { repo: entry.name }); + try { + await vi.advanceTimersByTimeAsync(0); + expect(mocks.search).not.toHaveBeenCalled(); + mocks.activeJob.progress.graphPhase = 'ready'; + mocks.activeJob.status = status; + await vi.advanceTimersByTimeAsync(1000); + expect(mocks.search).toHaveBeenCalledOnce(); + await pending; + expect(mocks.updateJob).not.toHaveBeenCalled(); + } finally { + mocks.activeJob.status = 'complete'; + await vi.advanceTimersByTimeAsync(1000); + await pending.catch(() => {}); + vi.useRealTimers(); + } + }, +); + describe('POST /api/embed staged recovery preflight', () => { - it('passes indexed source identity from the checkout and reports source failures', async () => { + it('hands the indexed storage slot to the shared phase and reports source failures', async () => { const fileHashes = { 'sample.ts': 'a'.repeat(64) }; mocks.loadMeta.mockResolvedValue({ repoPath: entry.path, @@ -163,20 +237,44 @@ describe('POST /api/embed staged recovery preflight', () => { contentRetention: 'full', stats: { embeddings: 3 }, }); - mocks.runEmbeddingPipeline.mockRejectedValue(new Error('source fingerprint mismatch')); - await invoke('/api/embed'); - await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); - expect(mocks.runEmbeddingPipeline.mock.calls[0]?.[6].source).toEqual({ - repoPath: entry.path, - fileHashes, - contentRetention: 'full', - }); - expect(entry.path).not.toBe(entry.storagePath); - expect(mocks.updateJob).toHaveBeenCalledWith( - 'embed-job', - expect.objectContaining({ status: 'failed', error: 'source fingerprint mismatch' }), - ); - expect(mocks.saveMeta).not.toHaveBeenCalled(); + const log = _captureLogger(); + try { + mocks.runAnalysisEmbeddingPhase.mockRejectedValueOnce( + new Error('source fingerprint mismatch', { + cause: new Error('private provider response excerpt'), + }), + ); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); + // The shared phase reads source identity from this slot's published metadata. + // embedding-job-controller.test.ts exercises that read with the real controller. + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + metadataDirectory: entry.storagePath, + graphPath: path.join(entry.storagePath, 'lbug'), + nodeLimit: 0, + resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), + signal: expect.any(AbortSignal), + }), + ); + expect(mocks.runEmbeddingPipeline).not.toHaveBeenCalled(); + expect(entry.path).not.toBe(entry.storagePath); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'failed', error: 'source fingerprint mismatch' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + expect(log.records()).toContainEqual( + expect.objectContaining({ + reason: 'source fingerprint mismatch', + repo: entry.name, + msg: 'Embedding generation failed', + }), + ); + expect(log.text()).not.toContain('private provider response excerpt'); + } finally { + log.restore(); + } }); afterEach(() => { @@ -312,18 +410,23 @@ describe('POST /api/embed staged recovery preflight', () => { expect.objectContaining({ status: 'complete' }), ); expect(mocks.ensurePrivateSharedGraph).toHaveBeenCalledTimes(1); - expect(mocks.withLbugDb).toHaveBeenCalledTimes(1); + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledTimes(1); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); expect(fs.existsSync(path.join(entry.storagePath, 'analyze.lock'))).toBe(false); }); }); -async function invoke(route: string, query: Record = {}) { +async function invoke( + route: string, + query: Record = {}, + body: Record = {}, +) { const layer = app.router.stack.find((item: any) => item.route?.path === route); expect(layer, route).toBeDefined(); const handler = layer.route.stack.at(-1).handle; const req = Object.assign(new EventEmitter(), { query, - body: { cypher: 'RETURN 1 AS value', query: 'handler', mode: 'bm25', enrich: false }, + body: { cypher: 'RETURN 1 AS value', query: 'handler', mode: 'bm25', enrich: false, ...body }, }); const res = Object.assign(new EventEmitter(), { statusCode: 200, @@ -433,8 +536,8 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => }); it.each(cases)( - 'preserves write mode while honoring $name metadata for embed', - async ({ fts, skip }) => { + 'keeps the $name graph out of the legacy writable DB path during embed', + async ({ fts }) => { mocks.loadMeta.mockResolvedValue({ capabilities: { fts } }); // This test stops at the DB boundary; it must not generate vectors or write an index. mocks.withLbugDb.mockResolvedValue(undefined); @@ -445,18 +548,82 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => expect.objectContaining({ status: 'complete' }), ), ); - expect(mocks.withLbugDb).toHaveBeenCalledExactlyOnceWith( - path.join(entry.storagePath, 'lbug'), - expect.any(Function), - skip ? { skipFts: true } : {}, + expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledWith( + expect.objectContaining({ + metadataDirectory: entry.storagePath, + graphPath: path.join(entry.storagePath, 'lbug'), + }), ); - expect(mocks.loadMeta).toHaveBeenCalledTimes(2); - expect(mocks.loadMeta).toHaveBeenNthCalledWith(1, entry.storagePath); - expect(mocks.loadMeta).toHaveBeenNthCalledWith(2, entry.storagePath); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); + expect(mocks.saveMeta).not.toHaveBeenCalled(); }, ); }); +describe('POST /api/embed published-phase outcomes', () => { + it('keeps graph readiness nonterminal until the shared phase completes', async () => { + mocks.loadMeta.mockResolvedValue({ repoPath: entry.path }); + let finish!: () => void; + mocks.runAnalysisEmbeddingPhase.mockImplementation(async ({ callbacks }) => { + callbacks.onProgress('graph-ready', 90, 'Graph available', { graphPhase: 'ready' }); + await new Promise((resolve) => { + finish = resolve; + }); + return { stats: { embeddings: 2 } }; + }); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ + progress: expect.objectContaining({ graphPhase: 'ready', phase: 'graph-ready' }), + }), + ); + expect(mocks.updateJob).not.toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.releaseIndexLock).not.toHaveBeenCalled(); + finish(); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + }); + + it('surfaces incomplete vectors without clearing durable work or reporting complete', async () => { + mocks.loadMeta.mockResolvedValue({ + repoPath: entry.path, + stats: { embeddings: 1 }, + embeddingJob: { identity: 'retained' }, + }); + mocks.runAnalysisEmbeddingPhase.mockImplementation(async ({ callbacks }) => { + callbacks.onProgress('embeddings', 98, 'Incomplete', { + graphPhase: 'ready', + embeddingProgress: { phase: 'failed', completedNodes: 1, pendingNodes: 2 }, + }); + throw new Error('Provider unavailable; resume durable progress'); + }); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ + status: 'failed', + error: expect.stringContaining('resume durable progress'), + }), + ); + expect(mocks.updateJob).not.toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); + }); +}); + describe('GET /api/search FTS warning redaction', () => { afterEach(() => { resetExtensionState(); @@ -490,3 +657,74 @@ describe('GET /api/repos catalog validation', () => { expect(mocks.listRegisteredRepos).toHaveBeenCalledTimes(1); }); }); + +it.each(['semantic', 'hybrid'])( + 'searches published vectors in %s mode without requiring a previously loaded model', + async (mode) => { + mocks.isEmbedderReady.mockReturnValue(true); + const metadata = { + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + }; + mocks.loadMeta.mockResolvedValue(metadata); + await invoke('/api/search', {}, { mode }); + expect(mocks.semanticSearch).toHaveBeenCalledOnce(); + const args = mocks.semanticSearch.mock.calls[0] as unknown[]; + expect(args[4]).toEqual({ + metadataDirectory: entry.storagePath, + metadata, + graphPath: path.join(entry.storagePath, 'lbug'), + }); + expect(args[5]).toBeTypeOf('function'); + // Only the verified vector reader invokes this lazy callback. + expect(mocks.embedQuery).not.toHaveBeenCalled(); + }, +); + +it.each(['semantic', 'hybrid'])( + 'degrades %s search when published vectors have no usable embedder', + async (mode) => { + mocks.loadMeta.mockResolvedValue({ + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'b'.repeat(64) }, + }); + const response = await invoke('/api/search', {}, { mode }); + expect(response.body.results).toEqual([]); + expect(mocks.semanticSearch).not.toHaveBeenCalled(); + expect(mocks.embedQuery).not.toHaveBeenCalled(); + expect(mocks.search).toHaveBeenCalledTimes(mode === 'hybrid' ? 1 : 0); + }, +); + +it('waits for embedding native cleanup before closing the backend or exiting on shutdown', async () => { + mocks.loadMeta.mockResolvedValue({ repoPath: entry.path }); + let finish!: () => void; + let signal!: AbortSignal; + mocks.runAnalysisEmbeddingPhase.mockImplementation(async (options) => { + signal = options.signal; + await new Promise((resolve) => { + finish = resolve; + }); + signal.throwIfAborted(); + }); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.runAnalysisEmbeddingPhase).toHaveBeenCalledOnce()); + const shutdown = process + .listeners('SIGTERM') + .find((listener) => !originalListeners.get('SIGTERM')!.includes(listener)); + expect(shutdown).toBeDefined(); + const stopped = (shutdown as unknown as () => Promise)(); + try { + await new Promise((resolve) => setImmediate(resolve)); + expect(signal.aborted).toBe(true); + expect(mocks.serverClose).toHaveBeenCalledOnce(); + expect(mocks.releaseIndexLock).not.toHaveBeenCalled(); + expect(mocks.disconnect).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + } finally { + finish(); + await stopped; + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledOnce()); + exit.mockRestore(); + } + expect(mocks.disconnect).toHaveBeenCalledOnce(); +}); diff --git a/gitnexus/test/unit/api-readonly-wiring.test.ts b/gitnexus/test/unit/api-readonly-wiring.test.ts index e53489966..32b7fff0f 100644 --- a/gitnexus/test/unit/api-readonly-wiring.test.ts +++ b/gitnexus/test/unit/api-readonly-wiring.test.ts @@ -67,29 +67,28 @@ describe('api read-only endpoint wiring', () => { ); }); - it('/api/embed refuses an in-place FTS crash WAL before the writable open', async () => { + it('/api/embed refuses an in-place FTS crash WAL before the vector phase', async () => { const source = await readSource(); const embedSection = source.match( - /\/\/ Run embedding pipeline asynchronously[\s\S]*?skipFtsOption\(ftsSession\.skipFts\)/, + /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, ); expect(embedSection).not.toBeNull(); const gateIdx = embedSection![0].indexOf('assertReadOnlyFtsCrashSafe(lbugPath)'); - const openIdx = embedSection![0].indexOf('await withLbugDb('); + const openIdx = embedSection![0].indexOf('await runAnalysisEmbeddingPhase('); expect(gateIdx).toBeGreaterThan(-1); expect(openIdx).toBeGreaterThan(gateIdx); expect(embedSection![0]).not.toMatch(/fts-inplace-checkpointed/); expect(embedSection![0]).not.toMatch(/readOnly:\s*true/); }); - it('/api/embed remains write-mode (writes embeddings — must not be flipped to readOnly)', async () => { + it('/api/embed delegates vector writes without reopening the graph in legacy write mode', async () => { const source = await readSource(); - // Negative assertion: no `readOnly: true` between the embed job's - // `runEmbeddingPipeline` call site and its withLbugDb open. Embed writes - // back vector rows; flipping this to readOnly would silently break it. - const embedSection = source.match(/runEmbeddingPipeline[\s\S]{0,400}\}\s*\)\s*;[\s\S]{0,200}/); - if (embedSection) { - expect(embedSection[0]).not.toMatch(/readOnly:\s*true/); - } + const embedSection = source.match( + /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, + ); + expect(embedSection).not.toBeNull(); + expect(embedSection![0]).toContain('await runAnalysisEmbeddingPhase('); + expect(embedSection![0]).not.toContain('await withLbugDb('); }); it('/api/embed keeps the repository lock until cancelled work actually stops', async () => { @@ -107,9 +106,9 @@ describe('api read-only endpoint wiring', () => { /\/\/ Run embedding pipeline asynchronously[\s\S]*?res\.status\(202\)/, ); expect(embedSection).not.toBeNull(); - expect(embedSection?.[0]).toContain('forceReembedNodeIds'); - expect(embedSection?.[0]).toContain('onCheckpointWindowStart'); - expect(embedSection?.[0]).toContain('pendingNodeIds'); - expect(embedSection?.[0]).toContain('saveMeta'); + expect(embedSection?.[0]).toContain('await runAnalysisEmbeddingPhase('); + expect(embedSection?.[0]).toContain('metadataDirectory: storagePath'); + expect(embedSection?.[0]).toContain('signal: embedController.signal'); + expect(embedSection?.[0]).not.toContain('embeddingCheckpoint: undefined'); }); }); diff --git a/gitnexus/test/unit/augmentation-generation.test.ts b/gitnexus/test/unit/augmentation-generation.test.ts new file mode 100644 index 000000000..d56bc1410 --- /dev/null +++ b/gitnexus/test/unit/augmentation-generation.test.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { branchSlug } from '../../src/storage/branch-index.js'; + +const mocks = vi.hoisted(() => ({ repos: vi.fn(), branch: vi.fn(), init: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', () => ({ listRegisteredRepos: mocks.repos })); +vi.mock('../../src/storage/storage-resolver.js', () => ({ + STATUS_STORAGE_REQUIREMENTS: {}, + requireRegisteredStoragePath: async (entry: { storagePath: string }) => entry.storagePath, +})); +vi.mock('../../src/storage/git.js', async (importOriginal) => ({ + ...(await importOriginal()), + getCurrentBranch: mocks.branch, +})); +vi.mock('../../src/core/lbug/pool-adapter.js', () => ({ + initLbug: mocks.init, + isLbugReady: () => false, + executeQuery: async () => [], +})); +vi.mock('../../src/core/search/bm25-index.js', () => ({ + searchFTSFromLbug: async () => ({ results: [], ftsAvailable: true }), +})); +import { augment } from '../../src/core/augmentation/engine.js'; + +let root: string; +beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-augment-generation-')); +}); +afterEach(async () => fs.rm(root, { recursive: true, force: true })); + +it.each([undefined, 'feature/generation'])( + 'opens the published graph in slot %s', + async (branch) => { + const storagePath = path.join(root, '.gitnexus'); + const slot = branch ? path.join(storagePath, 'branches', branchSlug(branch)) : storagePath; + const graphPath = `lbug.generation.${'a'.repeat(32)}`; + await fs.mkdir(slot, { recursive: true }); + await fs.writeFile(path.join(slot, 'gitnexus.json'), JSON.stringify({ graphPath })); + await fs.writeFile(path.join(slot, graphPath), 'owned graph fixture'); + mocks.branch.mockReturnValue(branch ?? 'main'); + mocks.repos.mockResolvedValue([ + { name: 'fixture', path: root, storagePath, branches: branch ? [{ branch }] : [] }, + ]); + + await augment('login', root); + + expect(mocks.init).toHaveBeenCalledWith('fixture', path.join(slot, graphPath)); + }, +); diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 7c88323cf..3575ca28b 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -1,18 +1,45 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, vi } from 'vitest'; -const { autoHeapCapMbMock } = vi.hoisted(() => ({ autoHeapCapMbMock: vi.fn(() => 512) })); -vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ - autoHeapCapMb: autoHeapCapMbMock, -})); - import { AutoSyncAnalysisError, - createAutoSyncAnalysisRunner, + createAutoSyncAnalysisRunner as createRunner, + type AutoSyncAnalysisRunner, } from '../../src/core/auto-sync/analysis-worker-launch.js'; +import { resolveAnalysisResourceBudget } from '../../src/core/analysis-resource-budget.js'; +import * as hostResources from '../../src/core/analysis-host-resources.js'; +import * as effectiveRam from '../../src/core/ingestion/utils/effective-ram.js'; + +function createAutoSyncAnalysisRunner(overrides: Parameters[0]) { + const run = createRunner(overrides); + return (...args: Parameters) => { + const slots = args[5] ?? 1; + args[1] = { + ...args[1], + resources: resolveAnalysisResourceBudget({ + effectiveMemoryBytes: (16 * 1024 ** 3) / slots, + availableCpus: Math.max(1, 8 / slots), + mainHeapMb: 2048 / slots, + modelMemoryMb: 0, + }), + }; + return run(...args); + }; +} + +const handoff = { + version: 1 as const, + publicationId: 'a'.repeat(32), + repoPath: '/tmp/repo', + storagePath: '/tmp/repo/.gitnexus', + metadataDirectory: '/tmp/repo/.gitnexus', + repoName: 'repo', + commit: 'b'.repeat(40), +}; function createChild() { return Object.assign(new EventEmitter(), { + pid: 1234, send: vi.fn(), stdout: { resume: vi.fn() }, stderr: { resume: vi.fn() }, @@ -20,6 +47,109 @@ function createChild() { } describe('auto-sync analysis worker', () => { + it.each([ + [1, 8192], + [4, 3072], + ])('admits the historical worker heap for %s concurrent repositories', async (slots, heap) => { + const cap = vi.spyOn(effectiveRam, 'autoHeapCapMb').mockReturnValue(12288); + const mayEmbed = vi.spyOn(hostResources, 'analysisMayEmbed').mockResolvedValue(false); + const admission = vi + .spyOn(hostResources, 'resolveHostAnalysisResources') + .mockImplementation((options) => + resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 32 * 1024 ** 3, + availableCpus: 8, + mainHeapMb: options.mainHeapMb, + modelMemoryMb: 0, + }), + ); + const child = createChild(); + const forkWorker = vi.fn(() => child as any); + const run = createRunner({ forkWorker }); + try { + const pending = run('/tmp/repo', {}, 30_000, undefined, undefined, slots); + await vi.waitFor(() => expect(forkWorker).toHaveBeenCalledOnce()); + expect(admission).toHaveBeenCalledWith( + expect.objectContaining({ mainHeapMb: heap + 48, concurrency: slots }), + ); + expect(forkWorker).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining([`--max-old-space-size=${heap}`]), + ); + child.emit('message', { type: 'complete', result: { stats: { files: 1 } } }); + child.emit('exit', 0, null); + await expect(pending).resolves.toEqual({ stats: { files: 1 } }); + } finally { + cap.mockRestore(); + mayEmbed.mockRestore(); + admission.mockRestore(); + } + }); + it('continues after actual exit and retains the original timeout across phases', async () => { + const graph = createChild(); + const embedding = createChild(); + const forkWorker = vi.fn().mockReturnValueOnce(graph).mockReturnValueOnce(embedding); + const clearTimeoutFn = vi.fn(); + const run = createAutoSyncAnalysisRunner({ + forkWorker, + setTimeoutFn: vi.fn(() => 1) as any, + clearTimeoutFn: clearTimeoutFn as any, + }); + const pending = run('/tmp/repo', {}, 50); + const prepared = { type: 'graph-prepared', handoff, result: { stats: { files: 3 } } }; + graph.emit('message', prepared); + expect(forkWorker).toHaveBeenCalledTimes(1); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + graph.emit('exit', 0, null); + expect(forkWorker).toHaveBeenCalledTimes(2); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + expect(embedding.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'continue', + prepared, + writerPid: graph.pid, + }), + ); + embedding.emit('message', { type: 'graph-ready' }); + expect(embedding.send).toHaveBeenLastCalledWith({ type: 'graph-visible' }); + let done = false; + void pending.then(() => { + done = true; + }); + embedding.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + await Promise.resolve(); + expect(done).toBe(false); + embedding.emit('exit', 0, null); + await expect(pending).resolves.toEqual({ stats: { files: 3 } }); + }); + + it('never starts a continuation after an unsuccessful graph exit', async () => { + const child = createChild(); + const forkWorker = vi.fn(() => child as any); + const run = createAutoSyncAnalysisRunner({ forkWorker }); + const pending = run('/tmp/repo', {}, 50); + child.emit('message', { type: 'graph-prepared', handoff, result: { stats: {} } }); + child.emit('exit', 1, null); + await expect(pending).rejects.toThrow('exited before completion'); + expect(forkWorker).toHaveBeenCalledTimes(1); + }); + + it('does not treat graph-prepared as terminal or clear the active timeout', async () => { + const child = createChild(); + const clearTimeoutFn = vi.fn(); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn(() => 1) as any, + clearTimeoutFn: clearTimeoutFn as any, + }); + const result = run('/tmp/repo', {}, 50); + child.emit('message', { type: 'graph-prepared', handoff, result: { stats: { files: 3 } } }); + expect(clearTimeoutFn).not.toHaveBeenCalled(); + child.emit('message', { type: 'error', message: 'cancelled after preparing' }); + child.emit('exit', 0, null); + await expect(result).rejects.toThrow('cancelled after preparing'); + }); + it('ignores progress and resolves from the terminal complete message', async () => { const child = createChild(); const forkWorker = vi.fn(() => child as any); @@ -28,7 +158,7 @@ describe('auto-sync analysis worker', () => { const result = run('/tmp/repo', { branch: 'main' }, 50); expect(forkWorker).toHaveBeenCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=512']), + expect.arrayContaining(['--max-old-space-size=2000']), ); child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); @@ -211,6 +341,41 @@ describe('auto-sync analysis worker', () => { expect(timers).toHaveLength(1); }); + it.each(['complete', 'error'] as const)( + 'preserves a terminal %s received before the deadline while waiting for exit', + async (type) => { + vi.useFakeTimers(); + try { + const child = createChild(); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: setTimeout, + clearTimeoutFn: clearTimeout, + }); + const result = run('/tmp/repo', {}, 50).catch((error: unknown) => error); + let settled = false; + void result.then(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(49); + child.emit( + 'message', + type === 'complete' + ? { type, result: { stats: { files: 3 } } } + : { type, message: 'worker failed' }, + ); + await vi.advanceTimersByTimeAsync(500); + expect(settled).toBe(false); + expect(child.send).toHaveBeenCalledTimes(1); + child.emit('exit', 0, null); + if (type === 'complete') await expect(result).resolves.toEqual({ stats: { files: 3 } }); + else await expect(result).resolves.toMatchObject({ message: 'worker failed' }); + } finally { + vi.useRealTimers(); + } + }, + ); + it('divides the worker heap by the number of repos analyzed in parallel', async () => { // Stubbed timers so neither run leaves a live timeout behind for the rest // of the suite, and both promises are settled before the test returns. @@ -233,13 +398,13 @@ describe('auto-sync analysis worker', () => { const parallel = run('/tmp/repo', { branch: 'main' }, 50, undefined, undefined, 4); expect(forkWorker).toHaveBeenLastCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=128']), + expect.arrayContaining(['--max-old-space-size=464']), ); const solo = run('/tmp/repo', { branch: 'main' }, 50); expect(forkWorker).toHaveBeenLastCalledWith( expect.any(String), - expect.arrayContaining(['--max-old-space-size=512']), + expect.arrayContaining(['--max-old-space-size=2000']), ); for (const child of forkChildren) { diff --git a/gitnexus/test/unit/chunker.test.ts b/gitnexus/test/unit/chunker.test.ts index af9a61c1d..aca797648 100644 --- a/gitnexus/test/unit/chunker.test.ts +++ b/gitnexus/test/unit/chunker.test.ts @@ -31,7 +31,8 @@ vi.mock('gitnexus-shared', async (importOriginal) => ({ getLanguageFromFilename, })); -import { chunkNode } from '../../src/core/embeddings/chunker.js'; +import { chunkNode, iterateNodeChunks } from '../../src/core/embeddings/chunker.js'; +import * as lineIndex from '../../src/core/embeddings/line-index.js'; type FakeNode = { type: string; @@ -250,6 +251,37 @@ describe('chunkNode', () => { expect(result[0].endLine).toBe(3); }); + it.each(['fixed', 'ast'] as const)( + 'projects %s class chunks only as the consumer requests them', + async (strategy) => { + const fields = Array.from({ length: 20 }, (_, i) => `field${i}: string;`); + const content = ['class Sample {', ...fields, '}'].join('\n'); + const tree = makeDeclarationTree('class_declaration', 'class_body', content, fields); + const expected = await chunkNode('Class', content, 'sample.ts', 10, 31, 40, 0, { + tree, + strategy, + }); + expect(expected.length).toBeGreaterThan(3); + const lines = vi.spyOn(lineIndex, 'resolveChunkLines'); + try { + const stream = iterateNodeChunks('Class', content, 'sample.ts', 10, 31, 40, 0, { + tree, + strategy, + }); + expect(lines).not.toHaveBeenCalled(); + const first = await stream.next(); + expect(first.value).toEqual(expected[0]); + // AST packing also resolves the projected line range before mapping it. + expect(lines.mock.calls.length).toBeLessThanOrEqual(2); + const actual = [first.value]; + for await (const chunk of stream) actual.push(chunk); + expect(actual).toEqual(expected); + } finally { + lines.mockRestore(); + } + }, + ); + it('preserves interface signatures via declaration-aware chunking', async () => { const content = [ 'interface Handler {', diff --git a/gitnexus/test/unit/clean-command-ownership.test.ts b/gitnexus/test/unit/clean-command-ownership.test.ts index de682fe89..70651a91e 100644 --- a/gitnexus/test/unit/clean-command-ownership.test.ts +++ b/gitnexus/test/unit/clean-command-ownership.test.ts @@ -173,4 +173,28 @@ describe('cleanCommand named branch empty-dir cleanup (#3331)', () => { await expect(fs.access(path.join(storagePath, 'branches'))).rejects.toBeTruthy(); await expect(fs.access(path.join(storagePath, 'gitnexus.json'))).resolves.toBeUndefined(); }); + + it.each(['branch', 'stale'])( + 'cleans a damaged generation pointer through %s without following it', + async (mode) => { + const { getStoragePaths, saveMeta } = await import('../../src/storage/repo-manager.js'); + const branchDir = path.dirname(getStoragePaths(repo, 'feature/x', storagePath).metaPath); + const outside = path.join(fixture.dbPath, 'external-sentinel'); + await fs.writeFile(outside, 'keep external data'); + const graphPath = 'lbug.generation.' + 'a'.repeat(32); + await fs.symlink(outside, path.join(branchDir, graphPath)); + await saveMeta(branchDir, { + repoPath: repo, + lastCommit: 'aaa', + indexedAt: '2026-09-20T00:00:00.000Z', + branch: 'feature/x', + graphPath, + }); + await cleanCommand( + mode === 'branch' ? { branch: 'feature/x', force: true } : { stale: true, force: true }, + ); + await expect(fs.access(branchDir)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await fs.readFile(outside, 'utf8')).toBe('keep external data'); + }, + ); }); diff --git a/gitnexus/test/unit/embedding-artifact.test.ts b/gitnexus/test/unit/embedding-artifact.test.ts new file mode 100644 index 000000000..188dfb85c --- /dev/null +++ b/gitnexus/test/unit/embedding-artifact.test.ts @@ -0,0 +1,221 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + adoptEmbeddingArtifact, + readEmbeddingArtifactSpace, + readEmbeddingArtifactOwners, + embeddingArtifactSpaceKey, + type EmbeddingArtifactOwner, + type EmbeddingArtifactRecord, +} from '../../src/storage/embedding-artifact.js'; +import * as records from '../../src/storage/embedding-job.js'; + +const space = { + vectorIdentity: 'a'.repeat(64), + inputIdentity: 'b'.repeat(64), + dimensions: 4, + schema: 'gitnexus-vector-store-v2', +}; +const owner = (digit: string): EmbeddingArtifactOwner => ({ + artifactIdentity: 'c'.repeat(64), + jobIdentity: digit.repeat(64), + graphGeneration: digit.repeat(32), + sourceIdentity: digit.repeat(64), + manifestDigest: digit.repeat(64), + phase: 'pending', + revalidateInputs: true, +}); +let root: string; +let validate: ReturnType Promise>>; +const directory = () => path.join(root, 'embeddings', 'spaces', embeddingArtifactSpaceKey(space)); +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-artifact-')); + validate = vi.fn(async () => undefined); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +describe('pending embedding artifact ownership', () => { + it('does not create ownership directories for a missing space', async () => { + expect(await readEmbeddingArtifactSpace(root, space, validate)).toBeNull(); + expect(await fs.readdir(root)).toEqual([]); + expect(validate).not.toHaveBeenCalled(); + }); + + it('ignores an empty interrupted space but retains unidentified contents', async () => { + await fs.mkdir(directory(), { recursive: true }); + expect(await readEmbeddingArtifactOwners(root)).toEqual([]); + await fs.writeFile(path.join(directory(), 'unknown'), 'retain'); + await expect(readEmbeddingArtifactOwners(root)).rejects.toThrow('no ownership receipt'); + expect(await fs.readFile(path.join(directory(), 'unknown'), 'utf8')).toBe('retain'); + }); + + it('keeps one artifact through repeated source generations without enumerating job history', async () => { + vi.spyOn(fs, 'readdir').mockRejectedValue(new Error('History scan')); + vi.spyOn(fs, 'opendir').mockRejectedValue(new Error('History scan')); + let record = await readEmbeddingArtifactSpace(root, space, validate); + expect(record).toBeNull(); + for (const digit of ['1', '2', '3']) { + record = await adoptEmbeddingArtifact(root, space, record, owner(digit), validate); + expect(record.owner.artifactIdentity).toBe('c'.repeat(64)); + expect((await readEmbeddingArtifactSpace(root, space, validate))?.owner.jobIdentity).toBe( + digit.repeat(64), + ); + } + expect(fs.readdir).not.toHaveBeenCalled(); + expect(fs.opendir).not.toHaveBeenCalled(); + }); + + it.each(['before-owner', 'after-owner'] as const)( + 'recovers a crash %s without losing ownership', + async (point) => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + if (point === 'before-owner') { + const write = records.writeEmbeddingJobRecord; + vi.spyOn(records, 'writeEmbeddingJobRecord').mockImplementation( + async (dir, name, payload) => { + if (name === 'space.json') throw new Error('Crash before owner'); + await write(dir, name, payload); + }, + ); + } else { + vi.spyOn(fs, 'unlink').mockRejectedValueOnce(new Error('Crash after owner')); + } + await expect( + adoptEmbeddingArtifact(root, space, first, owner('2'), validate), + ).rejects.toThrow('Crash'); + vi.restoreAllMocks(); + const recovered = await readEmbeddingArtifactSpace(root, space, validate); + expect(recovered?.owner).toEqual(owner('2')); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(recovered); + await expect(fs.stat(path.join(directory(), 'rebase.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }, + ); + + it('leaves the old owner unchanged if the new manifest cannot be verified', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + validate.mockRejectedValueOnce(new Error('Missing manifest')); + await expect(adoptEmbeddingArtifact(root, space, first, owner('2'), validate)).rejects.toThrow( + 'Missing manifest', + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('refuses a stale owner instead of overwriting a later adoption', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const second = await adoptEmbeddingArtifact(root, space, first, owner('2'), validate); + await expect(adoptEmbeddingArtifact(root, space, first, owner('3'), validate)).rejects.toThrow( + 'ownership changed', + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(second); + }); + + it('refuses an intent whose before and after owners have both been superseded', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const second: EmbeddingArtifactRecord = { ...first, owner: owner('2') }; + await records.writeEmbeddingJobRecord(directory(), 'rebase.json', { + version: 1, + before: first, + after: second, + }); + await records.writeEmbeddingJobRecord(directory(), 'space.json', { + ...first, + owner: owner('3'), + }); + await expect(readEmbeddingArtifactSpace(root, space, validate)).rejects.toThrow('superseded'); + expect(await records.readEmbeddingJobRecord(directory(), 'rebase.json')).toEqual({ + version: 1, + before: first, + after: second, + }); + }); + + it('requires input revalidation and keeps a pending paid artifact instead of replacing it', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + await expect( + adoptEmbeddingArtifact( + root, + space, + first, + { ...owner('2'), revalidateInputs: false }, + validate, + ), + ).rejects.toThrow('current-input validation'); + await expect( + adoptEmbeddingArtifact( + root, + space, + first, + { ...owner('2'), artifactIdentity: 'd'.repeat(64) }, + validate, + ), + ).rejects.toThrow('retained until sealed'); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('starts a new writer after sealing and retains the completed artifact as its sole fallback', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const sealed = await adoptEmbeddingArtifact( + root, + space, + first, + { ...first.owner, phase: 'sealed' }, + validate, + ); + await expect(adoptEmbeddingArtifact(root, space, sealed, owner('2'), validate)).rejects.toThrow( + 'Rebased artifact', + ); + const next = await adoptEmbeddingArtifact( + root, + space, + sealed, + { + ...owner('2'), + artifactIdentity: 'd'.repeat(64), + fallbackArtifact: first.owner.artifactIdentity, + }, + validate, + ); + expect(next.owner.fallbackArtifact).toBe(first.owner.artifactIdentity); + expect(next.owner.artifactIdentity).not.toBe(first.owner.artifactIdentity); + }); + + it('keeps incompatible input spaces independently owned and directly addressable', async () => { + const first = await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const other = { ...space, inputIdentity: 'd'.repeat(64) }; + expect(await readEmbeddingArtifactSpace(root, other, validate)).toBeNull(); + await adoptEmbeddingArtifact( + root, + other, + null, + { ...owner('2'), artifactIdentity: 'e'.repeat(64) }, + validate, + ); + expect(await readEmbeddingArtifactSpace(root, space, validate)).toEqual(first); + }); + + it('retains corrupt receipts and rejects a symlinked space directory', async () => { + await adoptEmbeddingArtifact(root, space, null, owner('1'), validate); + const file = path.join(directory(), 'space.json'); + await fs.writeFile(file, '{"checksum":"bad","payload":{}}'); + await expect(readEmbeddingArtifactSpace(root, space, validate)).rejects.toThrow( + 'checksum mismatch', + ); + expect(await fs.readFile(file, 'utf8')).toContain('"bad"'); + const other = { ...space, inputIdentity: 'e'.repeat(64) }; + await fs.symlink( + directory(), + path.join(root, 'embeddings', 'spaces', embeddingArtifactSpaceKey(other)), + process.platform === 'win32' ? 'junction' : 'dir', + ); + await expect(readEmbeddingArtifactSpace(root, other, validate)).rejects.toThrow( + 'owned directory', + ); + }); +}); diff --git a/gitnexus/test/unit/embedding-inputs-lazy.test.ts b/gitnexus/test/unit/embedding-inputs-lazy.test.ts new file mode 100644 index 000000000..00a6122f2 --- /dev/null +++ b/gitnexus/test/unit/embedding-inputs-lazy.test.ts @@ -0,0 +1,32 @@ +import { expect, it, vi } from 'vitest'; + +const registry = vi.hoisted(() => ({ loaded: false })); +vi.mock('../../src/core/ingestion/languages/index.js', () => { + registry.loaded = true; + throw new Error('The provider registry is unavailable in this fixture'); +}); + +import { iterateEmbeddingInputs } from '../../src/core/embeddings/embedding-inputs.js'; +import { DEFAULT_EMBEDDING_CONFIG } from '../../src/core/embeddings/types.js'; + +it('generates fixed function inputs without initializing language providers', async () => { + const content = 'export function value() { return 1; }'; + const inputs = []; + for await (const input of iterateEmbeddingInputs( + { + id: 'Function:value.ts:value', + label: 'Function', + name: 'value', + filePath: 'value.ts', + content, + startLine: 0, + endLine: 0, + }, + { ...DEFAULT_EMBEDDING_CONFIG, chunkStrategy: 'fixed' }, + )) { + inputs.push(input); + } + expect(inputs).toHaveLength(1); + expect(inputs[0].text).toContain(content); + expect(registry.loaded).toBe(false); +}); diff --git a/gitnexus/test/unit/embedding-job-file-security.test.ts b/gitnexus/test/unit/embedding-job-file-security.test.ts new file mode 100644 index 000000000..335cf5882 --- /dev/null +++ b/gitnexus/test/unit/embedding-job-file-security.test.ts @@ -0,0 +1,131 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { constants } from 'node:fs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; + +const directories: string[] = []; +async function fixture(payload: unknown): Promise { + const directory = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), 'gn-receipt-security-')), + ); + directories.push(directory); + await writeEmbeddingJobRecord(directory, 'job.json', payload); + return directory; +} + +afterEach(async () => { + vi.restoreAllMocks(); + vi.doUnmock('node:fs'); + vi.resetModules(); + await Promise.all( + directories.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })), + ); +}); + +it.each(['read', 'write'] as const)( + 'rejects a directory redirected during realpath before receipt %s', + async (operation) => { + const directory = await fixture({ owned: true }); + const outside = await fixture({ outside: true }); + const realpath = fs.realpath.bind(fs); + vi.spyOn(fs, 'realpath').mockImplementation((target, options) => + realpath(target === directory ? outside : target, options), + ); + const open = vi.spyOn(fs, 'open'); + await expect( + operation === 'read' + ? readEmbeddingJobRecord(directory, 'job.json') + : writeEmbeddingJobRecord(directory, 'job.json', { overwritten: true }), + ).rejects.toThrow('directory changed'); + expect(open).not.toHaveBeenCalled(); + expect(await readEmbeddingJobRecord(outside, 'job.json')).toEqual({ outside: true }); + }, +); + +it.skipIf(process.platform === 'win32')( + 'rejects a receipt replaced by a FIFO without blocking its open', + async () => { + const directory = await fixture({ owned: true }); + const file = path.join(directory, 'job.json'); + const open = fs.open.bind(fs); + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + if (target === file) { + await fs.unlink(file); + execFileSync('mkfifo', [file]); + // Fail before invoking the blocking syscall on the regressed version. + if (typeof flags !== 'number' || !(flags & constants.O_NONBLOCK)) + throw new Error('Unsafe blocking open of FIFO'); + } + return open(target, flags, mode); + }); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('regular file'); + }, +); + +it('rejects a different regular file at the opened path when O_NOFOLLOW is unavailable', async () => { + const directory = await fixture({ owned: true }); + const replacement = await fixture({ replacement: true }); + const file = path.join(directory, 'job.json'); + vi.doMock('node:fs', async (original) => { + const actual = await original(); + return { ...actual, constants: { ...actual.constants, O_NOFOLLOW: undefined } }; + }); + vi.resetModules(); + const { readEmbeddingJobRecord: readWithoutNoFollow } = + await import('../../src/storage/embedding-job.js'); + const open = fs.open.bind(fs); + const lstat = fs.lstat.bind(fs); + let close: ReturnType | undefined; + let read: ReturnType | undefined; + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + const handle = await open(target, flags, mode); + if (target === file) { + close = vi.spyOn(handle, 'close'); + read = vi.spyOn(handle, 'read'); + // Model a changed directory entry with real inode stats. Windows cannot + // rename over an open file, so a rename would fail before testing the guard. + vi.spyOn(fs, 'lstat').mockImplementation((target, options) => + lstat(target === file ? path.join(replacement, 'job.json') : target, options), + ); + } + return handle; + }); + await expect(readWithoutNoFollow(directory, 'job.json')).rejects.toThrow('changed during open'); + expect(close).toHaveBeenCalledOnce(); + expect(read).not.toHaveBeenCalled(); +}); + +it.skipIf(process.platform === 'win32')( + 'rejects a symlink swapped in during open when O_NOFOLLOW is unavailable', + async () => { + const directory = await fixture({ owned: true }); + const outside = await fixture({ outside: true }); + const file = path.join(directory, 'job.json'); + vi.doMock('node:fs', async (original) => { + const actual = await original(); + return { ...actual, constants: { ...actual.constants, O_NOFOLLOW: undefined } }; + }); + vi.resetModules(); + const { readEmbeddingJobRecord: readWithoutNoFollow } = + await import('../../src/storage/embedding-job.js'); + const open = fs.open.bind(fs); + let close: ReturnType | undefined; + vi.spyOn(fs, 'open').mockImplementation(async (target, flags, mode) => { + if (target === file) { + await fs.unlink(file); + await fs.symlink(path.join(outside, 'job.json'), file); + } + const handle = await open(target, flags, mode); + if (target === file) close = vi.spyOn(handle, 'close'); + return handle; + }); + await expect(readWithoutNoFollow(directory, 'job.json')).rejects.toThrow('regular file'); + expect(close).toHaveBeenCalledOnce(); + }, +); diff --git a/gitnexus/test/unit/embedding-job-journal.test.ts b/gitnexus/test/unit/embedding-job-journal.test.ts new file mode 100644 index 000000000..fb38aea3d --- /dev/null +++ b/gitnexus/test/unit/embedding-job-journal.test.ts @@ -0,0 +1,432 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + EmbeddingJobJournal, + embeddingInputFailureKey, + type EmbeddingUnitIntent, +} from '../../src/storage/embedding-job-journal.js'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; +import type { EmbeddingManifestReceipt } from '../../src/core/embeddings/embedding-work-manifest.js'; +import { embeddingJobIdentity } from '../../src/core/embeddings/embedding-job-identity.js'; + +const identity = 'a'.repeat(64); +const manifest: EmbeddingManifestReceipt = { + version: 1, + digest: 'b'.repeat(64), + sourceIdentity: 'c'.repeat(64), + bytes: 30, + totalNodes: 6, + totalUnits: 3, + maximumUnitBytes: 100, +}; +const unit = (n: number, origin: 'manifest' | 'failed' = 'manifest'): EmbeddingUnitIntent => ({ + start: { unit: n, offset: n * 10 }, + next: { unit: n + 1, offset: (n + 1) * 10 }, + nodes: 2, + origin, +}); +const directories: string[] = []; +async function collect(source: AsyncIterable): Promise { + const items: T[] = []; + for await (const item of source) items.push(item); + return items; +} +async function job() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-journal-')); + directories.push(directory); + return EmbeddingJobJournal.create(directory, identity, manifest); +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { force: true, recursive: true }); +}); + +describe('durable embedding scheduling', () => { + it.each(['Function:a.ts:bad', `Function:a.ts:${'x'.repeat(40_000)}`])( + 'retains rejection history across crashes for node ID %#', + async (id) => { + let journal = await job(); + await journal.begin(unit(0)); + for (let attempt = 1; attempt <= 3; attempt++) { + expect(await journal.recordInputRejections([id])).toEqual({ [id]: attempt }); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + expect(journal.snapshot().active).toMatchObject({ + inputFailureKeyFormat: 'sha256', + inputFailures: { [embeddingInputFailureKey(id)]: attempt }, + }); + if (attempt < 3) { + await journal.fail('provider'); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + } + } + await journal.acknowledge({ reusedNodes: 1, chunks: 1 }); + expect(journal.snapshot()).toMatchObject({ + excludedNodes: 1, + reusedNodes: 1, + completedNodes: 0, + }); + }, + ); + + it('upgrades existing raw-ID rejection history without resetting attempts', async () => { + let journal = await job(); + const id = 'Function:a.ts:legacy'; + await journal.begin({ ...unit(0), inputFailures: { [id]: 2 } }); + await journal.fail('input'); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + expect(await journal.recordInputRejections([id])).toEqual({ [id]: 3 }); + await journal.acknowledge({ reusedNodes: 1, chunks: 1 }); + expect(journal.snapshot().excludedNodes).toBe(1); + }); + + it('fits a full unit of long rejected IDs within the receipt limit', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-journal-wide-')); + directories.push(directory); + const wideManifest = { ...manifest, totalNodes: 64, totalUnits: 1, bytes: 10 }; + const journal = await EmbeddingJobJournal.create(directory, identity, wideManifest); + await journal.begin({ ...unit(0), nodes: 64 }); + const ids = Array.from({ length: 64 }, (_, i) => `Function:a.ts:${i}${'x'.repeat(40_000)}`); + await journal.recordInputRejections(ids); + const reopened = await EmbeddingJobJournal.open(journal.directory, identity, wideManifest); + expect(Object.keys(reopened.snapshot().active!.inputFailures!)).toHaveLength(64); + expect((await fs.stat(path.join(journal.directory, 'job.json'))).size).toBeLessThan(32 * 1024); + }); + + it.each([0, 4, 1.5])('rejects malformed persisted input failure count %s', async (count) => { + const journal = await job(); + await journal.begin(unit(0)); + const state = (await readEmbeddingJobRecord(journal.directory, 'job.json')) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...state, + active: { ...unit(0), inputFailures: { 'Function:a.ts:bad': count } }, + }); + await expect(EmbeddingJobJournal.open(journal.directory, identity, manifest)).rejects.toThrow(); + }); + + it('refuses a foreign failure record beside an active retry without deleting it', async () => { + const journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + await journal.begin(unit(0, 'failed')); + const name = 'failed-0000000000000000.json'; + const record = (await readEmbeddingJobRecord(journal.directory, name)) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, name, { ...record, identity: 'f'.repeat(64) }); + const before = await fs.readFile(path.join(journal.directory, name)); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toThrow('identity mismatch'); + expect(await fs.readFile(path.join(journal.directory, name))).toEqual(before); + }); + + it('keeps the journal usable after validation fails before any write', async () => { + const journal = await job(); + await expect(journal.indexing()).rejects.toThrow('accounting'); + await journal.begin(unit(0)); + await expect(journal.acknowledge({ reusedNodes: 3, chunks: 2 })).rejects.toThrow( + 'acknowledgement', + ); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + await expect(journal.fail('provider')).rejects.toThrow('Invalid embedding failure'); + await journal.begin(unit(1)); + expect(journal.snapshot()).toMatchObject({ reusedUnits: 1, active: unit(1) }); + }); + + it('requires reopening after a failed-unit write succeeds but its receipt fails', async () => { + const journal = await job(); + await journal.begin(unit(0)); + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + await rename(from, to); + if (path.basename(String(to)) === 'failed-0000000000000000.json') + throw new Error('injected failure after failed-unit rename'); + }); + await expect(journal.fail('provider')).rejects.toThrow('injected'); + await expect(journal.acknowledge({ reusedNodes: 0, chunks: 2 })).rejects.toThrow( + 'durability uncertain', + ); + }); + + it('refuses foreign failed-unit identity instead of resetting another job', async () => { + const journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + const name = 'failed-0000000000000000.json'; + const record = (await readEmbeddingJobRecord(journal.directory, name)) as Record< + string, + unknown + >; + await writeEmbeddingJobRecord(journal.directory, name, { ...record, identity: 'f'.repeat(64) }); + const before = await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8'); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toThrow('identity mismatch'); + expect(await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8')).toBe(before); + expect((await fs.readdir(journal.directory)).some((name) => name.includes('.preserved-'))).toBe( + false, + ); + }); + + it('propagates failure-record access errors without resetting valid accounting', async () => { + const journal = await job(); + const before = await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8'); + vi.spyOn(fs, 'opendir').mockRejectedValueOnce( + Object.assign(new Error('access denied'), { code: 'EACCES' }), + ); + await expect( + EmbeddingJobJournal.recover(journal.directory, identity, manifest), + ).rejects.toMatchObject({ code: 'EACCES' }); + expect(await fs.readFile(path.join(journal.directory, 'job.json'), 'utf8')).toBe(before); + }); + + it('retains an active retry whose old failed record was already removed', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + await journal.begin(unit(0, 'failed')); + await fs.unlink(path.join(journal.directory, 'failed-0000000000000000.json')); + journal = await EmbeddingJobJournal.recover(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ failedUnits: 0, active: unit(0, 'failed') }); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + expect(journal.snapshot()).toMatchObject({ reusedUnits: 1, active: null }); + }); + it('resumes only the active unit and keeps acknowledged counts', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.acknowledge({ reusedNodes: 0, chunks: 3 }); + await journal.begin(unit(1)); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ + completedUnits: 1, + completedNodes: 2, + cursor: { unit: 1, offset: 10 }, + active: unit(1), + chunks: 3, + }); + await journal.acknowledge({ reusedNodes: 2, chunks: 4 }); + await journal.begin(unit(2)); + await journal.acknowledge({ reusedNodes: 1, chunks: 2 }); + await journal.indexing(); + await journal.ready(); + expect( + (await EmbeddingJobJournal.open(journal.directory, identity, manifest)).snapshot(), + ).toMatchObject({ + phase: 'ready', + completedUnits: 2, + reusedUnits: 1, + completedNodes: 3, + reusedNodes: 3, + chunks: 9, + active: null, + }); + }); + + it('keeps failures retryable across arbitrarily many invocations', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + expect(await collect(journal.failed())).toEqual([]); + for (let attempt = 0; attempt < 5; attempt++) { + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(await collect(journal.failed())).toEqual([unit(0, 'failed')]); + await journal.begin(unit(0, 'failed')); + expect(journal.snapshot()).toMatchObject({ + failedUnits: 0, + failedNodes: 0, + active: unit(0, 'failed'), + }); + await journal.fail('provider'); + expect(journal.snapshot()).toMatchObject({ + failedUnits: 1, + failedNodes: 2, + cursor: { unit: 1, offset: 10 }, + }); + expect(await collect(journal.failed())).toEqual([]); + } + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + await journal.acknowledge({ reusedNodes: 0, chunks: 2 }); + expect(await collect(journal.failed())).toEqual([]); + expect(journal.snapshot()).toMatchObject({ completedUnits: 1, failedUnits: 0 }); + }); + + it('replays the active failed unit after its record was removed but before cursor acknowledgement', async () => { + let journal = await job(); + await journal.begin(unit(0)); + await journal.fail('provider'); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + await journal.begin(unit(0, 'failed')); + await fs.unlink(path.join(journal.directory, 'failed-0000000000000000.json')); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot().active).toEqual(unit(0, 'failed')); + await journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + expect(journal.snapshot()).toMatchObject({ failedUnits: 0, reusedUnits: 1 }); + }); + + it('reconciles an uncertain acknowledgement by reopening instead of counting twice', async () => { + let journal = await job(); + await journal.begin(unit(0)); + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + await rename(from, to); + if (String(to).endsWith('/job.json')) throw new Error('injected failure after rename'); + }); + await expect(journal.acknowledge({ reusedNodes: 0, chunks: 2 })).rejects.toThrow('injected'); + await expect(journal.begin(unit(1))).rejects.toThrow('durability uncertain'); + vi.restoreAllMocks(); + journal = await EmbeddingJobJournal.open(journal.directory, identity, manifest); + expect(journal.snapshot()).toMatchObject({ + completedUnits: 1, + completedNodes: 2, + active: null, + cursor: { unit: 1, offset: 10 }, + }); + await journal.begin(unit(1)); + }); + + it('rejects concurrent mutation instead of queuing unbounded intents', async () => { + const journal = await job(); + const first = journal.begin(unit(0)); + await expect(journal.begin(unit(0))).rejects.toThrow('Concurrent'); + await first; + expect(journal.snapshot().active).toEqual(unit(0)); + }); + + it.each(['identity', 'manifest', 'checksum', 'accounting', 'cursor'])( + 'retains artifacts on corrupt %s', + async (kind) => { + const journal = await job(); + await fs.writeFile(path.join(journal.directory, 'vectors'), 'retained'); + if (kind === 'checksum') + await fs.writeFile( + path.join(journal.directory, 'job.json'), + '{"checksum":"bad","payload":{}}', + ); + if (kind === 'accounting') + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...journal.snapshot(), + completedUnits: 1, + }); + if (kind === 'cursor') + await writeEmbeddingJobRecord(journal.directory, 'job.json', { + ...journal.snapshot(), + cursor: { unit: 0, offset: 9 }, + }); + await expect( + EmbeddingJobJournal.open( + journal.directory, + kind === 'identity' ? 'f'.repeat(64) : identity, + kind === 'manifest' ? { ...manifest, digest: 'd'.repeat(64) } : manifest, + ), + ).rejects.toThrow(); + expect(await fs.readFile(path.join(journal.directory, 'vectors'), 'utf8')).toBe('retained'); + }, + ); + + it('cannot publish incomplete coverage or mutate the live state through a snapshot', async () => { + const journal = await job(); + journal.snapshot().cursor.unit = 100; + expect(journal.snapshot().cursor.unit).toBe(0); + await expect(journal.indexing()).rejects.toThrow('accounting'); + expect(await readEmbeddingJobRecord(journal.directory, 'job.json')).toMatchObject({ + phase: 'embedding', + }); + }); + + it('validates failure filenames without accepting traversal', async () => { + const journal = await job(); + await expect( + writeEmbeddingJobRecord(journal.directory, '../failed-0000000000000000.json', {}), + ).rejects.toThrow('name'); + await expect(writeEmbeddingJobRecord(journal.directory, 'failed-0.json', {})).rejects.toThrow( + 'name', + ); + }); +}); + +describe('embedding job identity', () => { + const graph = { + generation: 'generation', + coveredContent: 'dirty-coverage', + revision: 'revision', + analyzer: 'analyzer', + parser: 'parser', + schema: 'schema', + vectorSchema: 'vectors-v2', + }; + const vector = { provider: 'local', model: 'model', dimensions: 384 }; + const config = { + chunkSize: 2000, + overlap: 100, + maxSnippetLength: 2000, + maxDescriptionLength: 500, + maxInputChars: 8192, + }; + it('excludes scheduling options and unrelated credential fields', () => { + const expanded = { + ...config, + batchSize: 1, + threads: 1, + resourceBudget: 1024, + apiKey: 'secret', + }; + expect(embeddingJobIdentity(graph, vector, 'v7', expanded)).toEqual( + embeddingJobIdentity(graph, vector, 'v7', config), + ); + }); + it('invalidates on each coverage and input dimension', () => { + const base = embeddingJobIdentity(graph, vector, 'v7', config); + for (const key of Object.keys(graph)) + expect( + embeddingJobIdentity({ ...graph, [key]: 'changed' }, vector, 'v7', config).identity, + ).not.toBe(base.identity); + for (const key of Object.keys(config)) + expect( + embeddingJobIdentity(graph, vector, 'v7', { + ...config, + [key]: config[key as keyof typeof config] + 1, + }).identity, + ).not.toBe(base.identity); + expect( + embeddingJobIdentity(graph, { ...vector, dimensions: 768 }, 'v7', config).identity, + ).not.toBe(base.identity); + expect( + embeddingJobIdentity(graph, { ...vector, model: 'other' }, 'v7', config).identity, + ).not.toBe(base.identity); + expect(() => + embeddingJobIdentity( + graph, + { ...vector, provider: 'https://user:secret@example.com' }, + 'v7', + config, + ), + ).toThrow(); + }); + it('separates AST/fixed layout and indexed retention without splitting equivalent defaults', () => { + const base = embeddingJobIdentity(graph, vector, 'v7', config); + expect(embeddingJobIdentity(graph, vector, 'v7', { ...config, chunkStrategy: 'ast' })).toEqual( + base, + ); + expect( + embeddingJobIdentity(graph, vector, 'v7', { ...config, chunkStrategy: 'fixed' }) + .inputIdentity, + ).not.toBe(base.inputIdentity); + for (const retention of ['symbol', 'none'] as const) + expect(embeddingJobIdentity(graph, vector, 'v7', config, retention).inputIdentity).not.toBe( + base.inputIdentity, + ); + }); +}); diff --git a/gitnexus/test/unit/embedding-job-progress.test.ts b/gitnexus/test/unit/embedding-job-progress.test.ts new file mode 100644 index 000000000..b86fc5169 --- /dev/null +++ b/gitnexus/test/unit/embedding-job-progress.test.ts @@ -0,0 +1,260 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { EmbeddingJobJournal } from '../../src/storage/embedding-job-journal.js'; +import { readEmbeddingJobProgress } from '../../src/storage/embedding-job-progress.js'; +import { writeEmbeddingJobRecord } from '../../src/storage/embedding-job.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { embeddingProgressFromState } from '../../src/core/analysis-progress.js'; +import { readIndexCapabilities } from '../../src/storage/index-capabilities.js'; + +const identity = 'a'.repeat(64); +const generation = 'b'.repeat(32); +const manifest = { + version: 1 as const, + sourceIdentity: 'c'.repeat(64), + digest: 'd'.repeat(64), + bytes: 30, + totalNodes: 6, + totalUnits: 3, + maximumUnitBytes: 100, +}; +const meta: RepoMeta = { + repoPath: '/repo', + lastCommit: 'head', + indexedAt: '', + graphGeneration: { version: 1, id: generation, coveredContent: 'e'.repeat(64) }, + embeddingJob: { version: 1, identity, graphGeneration: generation }, +}; +const unit = (n: number, origin: 'manifest' | 'failed' = 'manifest') => ({ + start: { unit: n, offset: n * 10 }, + next: { unit: n + 1, offset: (n + 1) * 10 }, + nodes: 2, + origin, +}); +const directories: string[] = []; +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-job-progress-')); + directories.push(root); + const directory = path.join(root, 'embeddings', identity); + await fs.mkdir(directory, { recursive: true }); + return { + root, + directory, + journal: await EmbeddingJobJournal.create(directory, identity, manifest), + }; +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('bounded durable embedding progress', () => { + it.each([42, {}, null])('reports malformed graph paths as unverified (%j)', async (graphPath) => { + const metadata = { + ...meta, + embeddingJob: undefined, + graphPath, + vectorGeneration: { preservedForGraph: generation }, + } as unknown as RepoMeta; + expect(await readEmbeddingJobProgress('/missing', metadata)).toEqual({ phase: 'unverified' }); + }); + + it.each(['interrupted', 'collapsed'] as const)( + 'rejects a legacy vector capability after an %s graph write', + async (state) => { + const legacy: RepoMeta = { + repoPath: '/repo', + indexedAt: '', + lastCommit: '', + incrementalInProgress: + state === 'interrupted' ? { startedAt: 1, toWriteCount: 1 } : undefined, + graphWriteCollapsed: state === 'collapsed' ? { expected: 100, persisted: 1 } : undefined, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'available' }, + vectorSearch: { + provider: 'ladybugdb-vector', + status: 'vector-index', + exactScanLimit: 10000, + }, + }, + }; + expect(await readIndexCapabilities('/missing', legacy)).toMatchObject({ + vectorSearch: { status: 'unavailable' }, + }); + }, + ); + + it('gates a persisted vector-index stamp before filesystem work when the new job is pending', async () => { + const f = await fixture(); + const previous: NonNullable = { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'available' }, + vectorSearch: { provider: 'ladybugdb-vector', status: 'vector-index', exactScanLimit: 10000 }, + }; + const stat = vi.spyOn(fs, 'lstat'); + expect(await readIndexCapabilities(f.root, { ...meta, capabilities: previous })).toMatchObject({ + graph: previous.graph, + fts: previous.fts, + vectorSearch: { status: 'unavailable', reason: 'embedding-pending' }, + }); + expect(stat).not.toHaveBeenCalled(); + expect( + await readIndexCapabilities('/missing', { + repoPath: '/repo', + indexedAt: '', + lastCommit: '', + capabilities: previous, + }), + ).toBe(previous); + }); + + it('reports missing published vector artifacts without opening a native database', async () => { + const f = await fixture(); + const published = { + ...meta, + embeddingJob: undefined, + vectorGeneration: { + version: 1 as const, + identity, + graphGeneration: generation, + vectorIdentity: 'c'.repeat(64), + inputIdentity: 'd'.repeat(64), + dimensions: 3, + nodes: 6, + chunks: 7, + indexKind: 'hnsw' as const, + }, + }; + expect(await readIndexCapabilities(f.root, published)).toMatchObject({ + vectorSearch: { status: 'unavailable', reason: 'vector-artifact-unavailable' }, + }); + expect( + await readIndexCapabilities(f.root, { + ...published, + vectorGeneration: { ...published.vectorGeneration, graphGeneration: 'f'.repeat(32) }, + }), + ).toMatchObject({ + vectorSearch: { status: 'unavailable', reason: 'vector-generation-mismatch' }, + }); + }); + it('does no filesystem work for legacy or graph-only metadata', async () => { + const stat = vi.spyOn(fs, 'lstat'); + expect(await readEmbeddingJobProgress('/missing', null)).toBeUndefined(); + expect( + await readEmbeddingJobProgress('/missing', { ...meta, embeddingJob: undefined }), + ).toBeUndefined(); + expect(stat).not.toHaveBeenCalled(); + }); + + it('counts durable completions, reuse, failures and active retries without double counting', async () => { + const f = await fixture(); + await f.journal.begin(unit(0)); + await f.journal.acknowledge({ reusedNodes: 1, chunks: 3 }); + await f.journal.begin(unit(1)); + await f.journal.fail('provider'); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 1, + failedUnits: 1, + activeUnit: null, + totalNodes: 6, + completedNodes: 1, + reusedNodes: 1, + pendingNodes: 2, + failedNodes: 2, + chunks: 3, + }); + await f.journal.begin(unit(1, 'failed')); + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + pendingUnits: 2, + failedUnits: 0, + activeUnit: 1, + pendingNodes: 4, + failedNodes: 0, + }); + await f.journal.acknowledge({ reusedNodes: 2, chunks: 2 }); + await f.journal.begin(unit(2)); + await f.journal.acknowledge({ reusedNodes: 0, chunks: 2 }); + await f.journal.indexing(); + await f.journal.ready(); + // Progress never certifies publication merely because the journal says ready. + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + phase: 'indexing', + completedUnits: 2, + reusedUnits: 1, + pendingUnits: 0, + failedUnits: 0, + completedNodes: 3, + reusedNodes: 3, + pendingNodes: 0, + failedNodes: 0, + }); + }); + + it('does not scan a manifest or source database to observe progress', async () => { + const f = await fixture(); + // No manifest.jsonl or vectors file exists. Observing a journal still works. + const open = vi.spyOn(fs, 'open'); + expect(await readEmbeddingJobProgress(f.root, meta)).toMatchObject({ + totalNodes: 6, + pendingNodes: 6, + }); + expect(open.mock.calls.map((call) => path.basename(String(call[0])))).toEqual(['job.json']); + }); + + it('reports missing, corrupt and oversized receipts as unverified, never zero work', async () => { + const f = await fixture(); + const file = path.join(f.directory, 'job.json'); + await fs.unlink(file); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await fs.writeFile(file, '{broken'); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await fs.writeFile(file, 'x'.repeat(33 * 1024)); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + }); + + it('refuses foreign graph identities and symlinked job ancestors', async () => { + const f = await fixture(); + expect( + await readEmbeddingJobProgress(f.root, { + ...meta, + embeddingJob: { version: 1, identity, graphGeneration: 'f'.repeat(32) }, + }), + ).toEqual({ phase: 'unverified' }); + const moved = path.join(f.root, 'preserved'); + await fs.rename(path.join(f.root, 'embeddings'), moved); + await fs.symlink( + moved, + path.join(f.root, 'embeddings'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + }); + + it('rejects impossible accounting and drops unknown journal fields', async () => { + const f = await fixture(); + await writeEmbeddingJobRecord(f.directory, 'job.json', { + ...f.journal.snapshot(), + completedUnits: 4, + }); + expect(await readEmbeddingJobProgress(f.root, meta)).toEqual({ phase: 'unverified' }); + await writeEmbeddingJobRecord(f.directory, 'job.json', { + ...f.journal.snapshot(), + sourcePath: '/secret', + token: 'private', + }); + expect(JSON.stringify(await readEmbeddingJobProgress(f.root, meta))).not.toMatch( + /secret|private|identity|manifest/, + ); + expect(() => embeddingProgressFromState({ ...f.journal.snapshot(), phase: 'ready' })).toThrow( + 'unverified', + ); + }); +}); diff --git a/gitnexus/test/unit/embedding-sidecar-client.test.ts b/gitnexus/test/unit/embedding-sidecar-client.test.ts index 7e6943419..9fad230f6 100644 --- a/gitnexus/test/unit/embedding-sidecar-client.test.ts +++ b/gitnexus/test/unit/embedding-sidecar-client.test.ts @@ -10,6 +10,7 @@ import type { SidecarRequest, SidecarResponse, } from '../../src/core/embeddings/embedding-sidecar-protocol.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; const embeddingsDir = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -144,6 +145,157 @@ describe('embedding sidecar client', () => { expect(forkMock).toHaveBeenCalledTimes(1); }); + it('reserves lazily, pins the admitted config, and reuses a compatible warm model', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const originalThreads = process.env.GITNEXUS_EMBEDDING_THREADS; + const originalNodeOptions = process.env.NODE_OPTIONS; + const first = await reserveEmbeddingSidecar(config, 512); + expect(forkMock).not.toHaveBeenCalled(); + await expect(reserveEmbeddingSidecar(config, 512)).rejects.toThrow('busy'); + try { + process.env.GITNEXUS_EMBEDDING_THREADS = '99'; + process.env.NODE_OPTIONS = '--max-old-space-size=4096'; + await first.initialize(); + const opts = forkMock.mock.calls[0][2]; + expect(opts.execArgv).toEqual( + expect.arrayContaining(['--max-old-space-size=512', '--max-semi-space-size=16']), + ); + expect(opts.env?.GITNEXUS_EMBEDDING_THREADS).toBeUndefined(); + expect(process.env.GITNEXUS_EMBEDDING_THREADS).toBe('99'); + expect(children[0].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: config, + }); + first.release(); + const second = await reserveEmbeddingSidecar(config, 512); + await second.initialize(); + await expect(second.embedBatch(['warm'])).resolves.toHaveLength(1); + second.release(); + expect(forkMock).toHaveBeenCalledTimes(1); + await expect(first.embedBatch(['released'])).rejects.toThrow('reservation was released'); + } finally { + first.release(); + if (originalThreads === undefined) delete process.env.GITNEXUS_EMBEDDING_THREADS; + else process.env.GITNEXUS_EMBEDDING_THREADS = originalThreads; + if (originalNodeOptions === undefined) delete process.env.NODE_OPTIONS; + else process.env.NODE_OPTIONS = originalNodeOptions; + } + }); + + it('pins model and chunk settings across environment changes before lazy initialization', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const lease = await reserveEmbeddingSidecar(config, 512); + const changed = { + MODEL_ID: 'different/model', + CHUNK_SIZE: '128', + CHUNK_OVERLAP: '32', + CHUNK_STRATEGY: 'fixed', + }; + try { + for (const [name, value] of Object.entries(changed)) + vi.stubEnv(`GITNEXUS_EMBEDDING_${name}`, value); + await lease.initialize(); + for (const name of Object.keys(changed)) + expect(forkMock.mock.calls[0][2].env?.[`GITNEXUS_EMBEDDING_${name}`]).toBeUndefined(); + expect(children[0].send.mock.calls[0][0]).toMatchObject({ embeddingConfig: config }); + } finally { + lease.release(); + vi.unstubAllEnvs(); + } + }); + + it('waits for an idle old process to exit before assigning a different budget', async () => { + const { reserveEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const first = await reserveEmbeddingSidecar(config, 512); + await first.initialize(); + first.release(); + children[0].kill.mockImplementation(() => true); + const next = reserveEmbeddingSidecar(config, 256); + await vi.waitFor(() => expect(children[0].kill).toHaveBeenCalled()); + expect(forkMock).toHaveBeenCalledTimes(1); + children[0].emit('close', null, 'SIGKILL'); + const second = await next; + await second.initialize(); + expect(forkMock).toHaveBeenCalledTimes(2); + expect(forkMock.mock.calls[1][2].execArgv).toContain('--max-old-space-size=256'); + second.release(); + }); + + it('keeps an aborted physical request reserved until the child actually responds', async () => { + const { reserveEmbeddingSidecar, drainEmbeddingSidecarForAnalysis } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const lease = await reserveEmbeddingSidecar(config, 512); + await lease.initialize(); + children[0].send = vi.fn(() => true); + const abort = new AbortController(); + const pendingBatch = lease.embedBatch(['still running'], { signal: abort.signal }); + await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled()); + const request = children[0].send.mock.calls[0][0]; + abort.abort(); + await expect(pendingBatch).rejects.toThrow(); + lease.release(); + await expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow('busy'); + await expect(drainEmbeddingSidecarForAnalysis()).rejects.toThrow('busy'); + expect(children[0].kill).not.toHaveBeenCalled(); + children[0].emit('message', { type: 'vectors', id: request.id, vectors: [[0.25, 0.5, 0.75]] }); + await drainEmbeddingSidecarForAnalysis(); + expect(children[0].kill).toHaveBeenCalledTimes(1); + }); + + it('does not certify a timed-out reap or allocate another job over the retained process', async () => { + vi.useFakeTimers(); + const { reserveEmbeddingSidecar, ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1 }); + const first = await reserveEmbeddingSidecar(config, 512); + await first.initialize(); + first.release(); + children[0].kill.mockImplementation(() => true); + const rejected = expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow( + 'native resource reservation', + ); + await vi.advanceTimersByTimeAsync(5_001); + await rejected; + await expect(ensureEmbeddingSidecar()).rejects.toThrow('native resource reservation'); + await expect(reserveEmbeddingSidecar(config, 256)).rejects.toThrow( + 'native resource reservation', + ); + expect(forkMock).toHaveBeenCalledTimes(1); + children[0].emit('close', null, 'SIGKILL'); + const second = await reserveEmbeddingSidecar(config, 256); + second.release(); + }); + + it('rejects config changes while a job owns the runtime and preserves its allocation on respawn', async () => { + const { reserveEmbeddingSidecar, ensureEmbeddingSidecar, drainEmbeddingSidecarForAnalysis } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const config = resolveEmbeddingConfig({ threads: 1, device: 'cpu' }); + const lease = await reserveEmbeddingSidecar(config, 512); + await expect(ensureEmbeddingSidecar({ forceDevice: 'cuda' })).rejects.toThrow('conflicts'); + expect(forkMock).not.toHaveBeenCalled(); + await lease.initialize(); + await expect(ensureEmbeddingSidecar({ embeddingConfig: { threads: 2 } })).rejects.toThrow( + 'conflicts', + ); + await expect(drainEmbeddingSidecarForAnalysis()).rejects.toThrow('busy'); + children[0].emit('close', 1, null); + await expect(lease.embedBatch(['retry'])).resolves.toHaveLength(1); + expect(forkMock.mock.calls[1][2].execArgv).toContain('--max-old-space-size=512'); + expect(children[1].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: config, + }); + lease.release(); + await drainEmbeddingSidecarForAnalysis(); + }); + it('does not fork on darwin/x64', async () => { const orig = { platform: process.platform, arch: process.arch }; Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); diff --git a/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts b/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts new file mode 100644 index 000000000..ed23c8296 --- /dev/null +++ b/gitnexus/test/unit/embedding-sidecar-lifetime.test.ts @@ -0,0 +1,30 @@ +import { spawn } from 'node:child_process'; +import { expect, it } from 'vitest'; + +it('exits the standalone model process when its owner disconnects', async () => { + const entry = new URL('../../dist/core/embeddings/embedding-sidecar.js', import.meta.url).href; + const child = spawn( + process.execPath, + ['--input-type=module', '-e', `await import(${JSON.stringify(entry)}); process.send('ready');`], + { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + }, + ); + let stderr = ''; + child.stderr?.on('data', (data) => { + stderr += String(data); + }); + const exit = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', resolve); + }); + const ready = new Promise((resolve) => child.once('message', resolve)); + try { + expect(await Promise.race([ready, exit])).toBe('ready'); + child.disconnect(); + expect(await exit, stderr).toBe(1); + } finally { + if (child.connected) child.disconnect(); + await exit; + } +}); diff --git a/gitnexus/test/unit/embedding-source-reader.test.ts b/gitnexus/test/unit/embedding-source-reader.test.ts index 4531d661f..6b308c98f 100644 --- a/gitnexus/test/unit/embedding-source-reader.test.ts +++ b/gitnexus/test/unit/embedding-source-reader.test.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { mkdtemp, mkdir, writeFile, rm, symlink, unlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { EmbeddingSourceReader, needsEmbeddingSource, @@ -38,6 +38,40 @@ async function file(name: string, text: string) { } describe('original embedding source', () => { + it('admits source bytes before reading, including cached files', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source); + const admit = vi.fn(async (): Promise => { + throw new Error('budget exhausted'); + }); + await expect(reader.read(node('sample.py', 0, 0), admit)).rejects.toThrow('budget exhausted'); + expect(admit).toHaveBeenCalledWith(6); + admit.mockImplementation(async () => {}); + expect(await reader.read(node('sample.py', 0, 0), admit)).toBe('source'); + await unlink(path.join(root, 'sample.py')); + expect(await reader.read(node('sample.py', 0, 0), admit)).toBe('source'); + expect(admit).toHaveBeenCalledTimes(3); + }); + + it('rejects growth after admission instead of reading beyond the reservation', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source); + await expect( + reader.read(node('sample.py', 0, 0), async () => { + await writeFile(path.join(root, 'sample.py'), 'source grew'); + }), + ).rejects.toThrow('source size changed'); + }); + + it('honors the job-specific cache allowance', async () => { + await file('sample.py', 'source'); + const reader = new EmbeddingSourceReader(source, 8); + expect(await reader.read(node('sample.py', 0, 0))).toBe('source'); + await unlink(path.join(root, 'sample.py')); + await expect(reader.read(node('sample.py', 0, 0))).rejects.toThrow( + 'Embedding source unavailable', + ); + }); it('retains indentation, Unicode and exact zero-based inclusive CRLF lines', async () => { const text = '# prefix\r\ndef sample():\r\n value = "雪😀"\r\n return value\r\n# suffix'; await file('sample.py', text); diff --git a/gitnexus/test/unit/embedding-streaming-text.test.ts b/gitnexus/test/unit/embedding-streaming-text.test.ts new file mode 100644 index 000000000..2fb09ced7 --- /dev/null +++ b/gitnexus/test/unit/embedding-streaming-text.test.ts @@ -0,0 +1,222 @@ +import { createHash } from 'node:crypto'; +import { describe, expect, it, vi } from 'vitest'; +import { iterateCharacterChunks } from '../../src/core/embeddings/character-chunk.js'; +import { buildLineIndex } from '../../src/core/embeddings/line-index.js'; +import { contentHashForNode } from '../../src/core/embeddings/embedding-pipeline.js'; +import { extractDeclarationOnly } from '../../src/core/embeddings/text-generator.js'; +import { iterateEmbeddingInputs } from '../../src/core/embeddings/embedding-inputs.js'; +import { resolveEmbeddingConfig } from '../../src/core/embeddings/config.js'; + +const fingerprintNode = (label: string, content: string) => ({ + id: 'Function:src/example.ts:test', + name: 'Example', + filePath: 'src/example.ts', + description: 'Sample description.', + startLine: 1, + endLine: 99, + label, + content, +}); + +describe('bounded embedding text preserves existing vectors', () => { + // Exact text AND line-range receipts captured from AST base 6895f91726a3. + it.each([ + [ + 'Function', + 'sample.ts', + 'ast', + 'full', + 12, + 'e3728ed50afd80c3d9da4ba1b2d23c3f906b230851261462e0e554623196f457', + ], + [ + 'Function', + 'sample.ts', + 'fixed', + 'full', + 15, + '718690efd94d6ec0ff458b80280751b16aa57ad285ea5541bbe4f0128ab0999d', + ], + [ + 'Class', + 'sample.ts', + 'ast', + 'full', + 1, + '3af1649850519626a030297788391a745ee4370269f1f44a33f4362f7f38f5c2', + ], + [ + 'Class', + 'sample.ts', + 'fixed', + 'full', + 1, + '3af1649850519626a030297788391a745ee4370269f1f44a33f4362f7f38f5c2', + ], + [ + 'Function', + 'sample.ts', + 'ast', + 'symbol', + 12, + '8c5780484a0f3cfcfbd8b010ecb6ea1988f70594cbc10b400e7d9e8836e9f5c0', + ], + [ + 'Function', + 'sample.ts', + 'ast', + 'none', + 12, + '8c5780484a0f3cfcfbd8b010ecb6ea1988f70594cbc10b400e7d9e8836e9f5c0', + ], + [ + 'Function', + 'sample.ipynb', + 'ast', + 'full', + 12, + '04670833ee0a82f4e243bf065320bed8a6d2218168b7be617eabe6db144e8eb3', + ], + ] as const)( + 'matches AST-base %s %s %s %s inputs', + async (label, filePath, chunkStrategy, contentRetention, count, digest) => { + const content = + label === 'Class' + ? 'class Sample {\n field = 42;\n run() { return "BODY_EXCLUDED"; }\n}' + : filePath.endsWith('ipynb') + ? 'def Sample():\n' + + Array.from({ length: 45 }, (_, i) => ` value${i} = "雪😀"`).join('\n') + + '\n return value44' + : 'function Sample() {\n' + + Array.from({ length: 45 }, (_, i) => ` const value${i} = "雪😀";`).join('\n') + + '\n return value44;\n}'; + const node = { + id: `${label}:${filePath}:Sample`, + name: 'Sample', + label, + filePath, + content, + startLine: 10, + endLine: 10 + content.split('\n').length - 1, + }; + const config = resolveEmbeddingConfig({ + chunkSize: 100, + overlap: 20, + maxInputChars: 256, + chunkStrategy, + }); + const release = vi.fn(); + const readSource = vi.fn(async () => ({ content, release })); + const inputs = []; + for await (const input of iterateEmbeddingInputs(node, config, { + source: { repoPath: '/unused', fileHashes: {}, contentRetention }, + readSource, + })) + inputs.push(input); + expect(inputs).toHaveLength(count); + expect(createHash('sha256').update(JSON.stringify(inputs)).digest('hex')).toBe(digest); + expect(release).toHaveBeenCalledTimes(contentRetention === 'full' ? 1 : 0); + expect(node).not.toHaveProperty('classDeclaration'); + expect( + inputs + .filter(() => label === 'Class') + .map((input) => input.text) + .join('\n'), + ).not.toContain('BODY_EXCLUDED'); + }, + ); + + it('releases expanded source when iteration is cancelled at a chunk boundary', async () => { + const node = { + ...fingerprintNode('Function', 'function sample() ... [truncated]'), + name: 'sample', + }; + const release = vi.fn(); + const iterator = iterateEmbeddingInputs(node, resolveEmbeddingConfig({}), { + source: { repoPath: '/unused', fileHashes: {}, contentRetention: 'full' }, + readSource: async () => ({ + content: 'function sample() {\n' + ' consume();\n'.repeat(1000) + '}', + release, + }), + }); + expect((await iterator.next()).done).toBe(false); + expect(release).not.toHaveBeenCalled(); + await iterator.return(undefined); + expect(release).toHaveBeenCalledOnce(); + }); + it.each(['short', 'x'.repeat(2401)])( + 'rejects a negative overlap without skipping source text (case %#)', + (content) => { + expect(() => [...iterateCharacterChunks(content, 1, 1, 1200, -1)]).toThrow('overlap'); + }, + ); + + it.each([ + [ + 'Function', + 'export function 漢() {\r\n return "😀"; \r\n\r\n\r\n}\u3000\n', + '69f18eaa2f90496499546d329389cfbbaf05b5ed', + ], + ['File', 'a \t\r\n\n\n'.repeat(100000), 'bcdad0b0fb7dda9b85406be8c775ef1df3ac060a'], + [ + 'Class', + 'export class Example {\n value: string;\n method() { return "😀"; }\n}\n', + '05ce2f0a472c683379947bfe495db27a24e9c401', + ], + [ + 'TypeAlias', + 'type Huge = ' + '"漢😀" | '.repeat(40000) + 'never;\n', + 'b0b393e258779fafede851f76d33d0831695e6bb', + ], + // Captured independently from AST base 6895f91726a3fc2f82add44c4bb0287a5c6b28d3. + ])('keeps the AST-base %s fingerprint', (label, content, expected) => { + expect(contentHashForNode(fingerprintNode(label, content))).toBe(expected); + }); + + it('retains every character and exact line spans across Unicode and unusual overlaps', () => { + const content = '\n😀漢\r\n a\n'.repeat(70); + for (const overlap of [0, 1, 7, 8, 20]) { + let covered = 0; + for (const chunk of iterateCharacterChunks(content, 13, 999, 8, overlap)) { + expect(chunk.text).toBe(content.slice(chunk.startOffset, chunk.endOffset)); + expect(chunk.startLine).toBe( + 13 + content.slice(0, chunk.startOffset).split('\n').length - 1, + ); + expect(chunk.endLine).toBe( + 13 + content.slice(0, chunk.endOffset - 1).split('\n').length - 1, + ); + expect(chunk.startOffset).toBeLessThanOrEqual(covered); + covered = chunk.endOffset; + } + expect(covered).toBe(content.length); + } + }); + + it('produces the first huge-line chunk without retaining the remaining chunks', () => { + const content = '漢😀'.repeat(1000000); + const iterator = iterateCharacterChunks(content, 1, 1, 1200, 120); + expect(iterator.next().value).toMatchObject({ + text: content.slice(0, 1200), + startOffset: 0, + endOffset: 1200, + startLine: 1, + endLine: 1, + }); + iterator.return(undefined); + expect(iterator.next().done).toBe(true); + }); + + it('packs line offsets including the final empty line', () => { + expect(buildLineIndex('a\r\n😀\n\n')).toEqual(new Int32Array([0, 3, 6, 7])); + const index = buildLineIndex('a\n'.repeat(1000000)); + expect(index.length).toBe(1000001); + expect(index.at(-1)).toBe(2000000); + }); + + it('keeps exactly three lines of declaration brace lookahead', () => { + expect(extractDeclarationOnly('class Example\n\n\n{\n field: string;\n}')).toBe( + 'class Example', + ); + expect(extractDeclarationOnly('class Example\n\n\n\n{\n field: string;\n}')).toBe(''); + }); +}); diff --git a/gitnexus/test/unit/embedding-work-manifest.test.ts b/gitnexus/test/unit/embedding-work-manifest.test.ts new file mode 100644 index 000000000..c42c7b1b9 --- /dev/null +++ b/gitnexus/test/unit/embedding-work-manifest.test.ts @@ -0,0 +1,261 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { EmbeddingByteBudget } from '../../src/core/analysis-resource-budget.js'; +import { + createEmbeddingWorkManifest, + fetchEmbeddingWorkUnit, + loadEmbeddingWorkManifest, + readEmbeddingWorkUnits, + type EmbeddingManifestSource, + type EmbeddingWorkKey, +} from '../../src/core/embeddings/embedding-work-manifest.js'; +import type { EmbeddableNode } from '../../src/core/embeddings/types.js'; +import { + readEmbeddingJobRecord, + writeEmbeddingJobRecord, +} from '../../src/storage/embedding-job.js'; + +const directories: string[] = []; +const temporary = async (): Promise => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-work-manifest-')); + directories.push(directory); + return directory; +}; +afterEach(async () => { + await Promise.all( + directories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +function synthetic( + count: number, + contentBytes = 1024, +): EmbeddingManifestSource & { scans: number; fetches: number; largestFetch: number } { + return { + scans: 0, + fetches: 0, + largestFetch: 0, + async *scan(label: string): AsyncGenerator { + this.scans++; + if (label !== 'Function') return; + for (let i = 0; i < count; i++) + yield { + id: `Function:f.ts:fn${String(i).padStart(8, '0')}`, + contentBytes, + metadataBytes: 1024, + }; + }, + async fetch(label: string, ids: readonly string[]): Promise { + this.fetches++; + this.largestFetch = Math.max(this.largestFetch, ids.length); + return ids + .map((id) => ({ + id, + name: id, + label, + filePath: 'f.ts', + content: 'x'.repeat(contentBytes), + startLine: 1, + endLine: 1, + })) + .reverse(); + }, + }; +} + +describe('durable embedding selection', () => { + it('resumes by cursor without rescanning or refetching completed nodes', async () => { + const directory = await temporary(); + const source = synthetic(130); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + expect(receipt.totalNodes).toBe(130); + expect(receipt.totalUnits).toBe(3); + const budget = new EmbeddingByteBudget(512 * 1024); + const units = readEmbeddingWorkUnits(directory, receipt); + const first = await units.next(); + if (first.done) throw new Error('Missing first unit'); + const fetched = await fetchEmbeddingWorkUnit(source, first.value.unit, budget); + expect(fetched.nodes[0].id).toBe(first.value.unit.nodes[0].id); + fetched.release(); + await units.return(undefined); + const scans = source.scans; + const reloaded = await loadEmbeddingWorkManifest(directory, 'graph-a'); + const remaining: number[] = []; + for await (const item of readEmbeddingWorkUnits(directory, reloaded, first.value.next)) { + remaining.push(item.unit.ordinal); + const loaded = await fetchEmbeddingWorkUnit(source, item.unit, budget); + expect(loaded.nodes.some((node) => node.id === first.value.unit.nodes[0].id)).toBe(false); + loaded.release(); + } + expect(remaining).toEqual([1, 2]); + expect(source.scans).toBe(scans); + expect(source.fetches).toBe(3); + expect(budget.usedBytes).toBe(0); + }); + + it('bounds fetched payloads independently of a lazily generated corpus size', async () => { + const peaks: number[] = []; + for (const count of [10_000, 100_000, 1_000_000]) { + const directory = await temporary(); + const source = synthetic(count); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + const budget = new EmbeddingByteBudget(512 * 1024); + let read = 0; + for await (const { unit } of readEmbeddingWorkUnits(directory, receipt)) { + const fetched = await fetchEmbeddingWorkUnit(source, unit, budget); + read += fetched.nodes.length; + fetched.release(); + } + expect(read).toBe(count); + expect(source.largestFetch).toBe(64); + expect(budget.usedBytes).toBe(0); + peaks.push(budget.peakBytes); + } + expect(peaks).toEqual([256 * 1024, 256 * 1024, 256 * 1024]); + }, 30_000); + + it('gives an oversized node its own stable unit and admits it explicitly', async () => { + const directory = await temporary(); + const source = synthetic(2, 2 * 1024 * 1024); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + expect(receipt.totalUnits).toBe(2); + for await (const { unit } of readEmbeddingWorkUnits(directory, receipt)) { + const before = source.fetches; + await expect( + fetchEmbeddingWorkUnit(source, unit, new EmbeddingByteBudget(1024)), + ).rejects.toThrow('requires'); + expect(source.fetches).toBe(before); + const fetched = await fetchEmbeddingWorkUnit( + source, + unit, + new EmbeddingByteBudget(5 * 1024 * 1024), + ); + expect(fetched.nodes[0].content.length).toBe(2 * 1024 * 1024); + fetched.release(); + } + }); + + it('rejects cross-label duplicate identities before publishing a selection', async () => { + const directory = await temporary(); + const source = synthetic(0); + source.scan = async function* (label) { + if (label === 'Function' || label === 'Class') + yield { id: 'shared-id', contentBytes: 1, metadataBytes: 1024 }; + }; + await expect(createEmbeddingWorkManifest(directory, 'graph-a', source)).rejects.toThrow( + 'duplicate graph node identity', + ); + expect(source.fetches).toBe(0); + await expect(fs.stat(path.join(directory, 'manifest.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('does not substitute File fallback when a symbol scan fails', async () => { + const directory = await temporary(); + const labels: string[] = []; + const source = synthetic(0); + source.scan = async function* (label) { + labels.push(label); + throw new Error('graph read failed'); + }; + await expect(createEmbeddingWorkManifest(directory, 'graph-a', source)).rejects.toThrow( + 'graph read failed', + ); + expect(labels).not.toContain('File'); + }); + + it('selects File only after every symbol label is empty', async () => { + const directory = await temporary(); + const source = synthetic(0); + source.scan = async function* (label) { + if (label === 'File') yield { id: 'File:README.md', contentBytes: 12, metadataBytes: 1024 }; + }; + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + for await (const result of readEmbeddingWorkUnits(directory, receipt)) { + expect(result.unit.label).toBe('File'); + break; + } + expect(receipt.totalNodes).toBe(1); + }); + + it('rejects source drift and releases its reserved capacity on a failed fetch', async () => { + const directory = await temporary(); + const source = synthetic(1); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', source); + const iterator = readEmbeddingWorkUnits(directory, receipt); + const first = await iterator.next(); + if (first.done) throw new Error('Missing unit'); + await iterator.return(undefined); + const budget = new EmbeddingByteBudget(4096); + source.fetch = async () => []; + await expect(fetchEmbeddingWorkUnit(source, first.value.unit, budget)).rejects.toThrow( + 'missing or duplicate', + ); + expect(budget.usedBytes).toBe(0); + source.fetch = async () => [ + { + id: first.value.unit.nodes[0].id, + label: 'Function', + name: 'f', + filePath: 'f.ts', + content: 'changed', + }, + ]; + await expect(fetchEmbeddingWorkUnit(source, first.value.unit, budget)).rejects.toThrow( + 'graph changed', + ); + expect(budget.usedBytes).toBe(0); + }); + + it('preserves and rejects corrupt or foreign selections', async () => { + const directory = await temporary(); + await createEmbeddingWorkManifest(directory, 'graph-a', synthetic(1)); + await expect(loadEmbeddingWorkManifest(directory, 'graph-b')).rejects.toThrow('identity'); + await fs.appendFile(path.join(directory, 'manifest.jsonl'), '{}\n'); + await expect(loadEmbeddingWorkManifest(directory, 'graph-a')).rejects.toThrow('size mismatch'); + expect((await fs.stat(path.join(directory, 'manifest.jsonl'))).size).toBeGreaterThan(0); + }); + + it('rejects a misaligned durable cursor', async () => { + const directory = await temporary(); + const receipt = await createEmbeddingWorkManifest(directory, 'graph-a', synthetic(65)); + await expect( + readEmbeddingWorkUnits(directory, receipt, { unit: 1, offset: 0 }).next(), + ).rejects.toThrow('Invalid embedding work unit'); + }); +}); + +describe('compact embedding receipts', () => { + it.skipIf(process.platform === 'win32')( + 'refuses symlink receipts and job directories', + async () => { + const directory = await temporary(); + const outside = await temporary(); + await writeEmbeddingJobRecord(outside, 'job.json', { cursor: 1 }); + await fs.symlink(path.join(outside, 'job.json'), path.join(directory, 'job.json')); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('regular file'); + await fs.symlink(outside, path.join(directory, 'linked-job')); + await expect( + readEmbeddingJobRecord(path.join(directory, 'linked-job'), 'job.json'), + ).rejects.toThrow('owned directory'); + }, + ); + + it('checks the checksum and bounds record sizes', async () => { + const directory = await temporary(); + await writeEmbeddingJobRecord(directory, 'job.json', { cursor: 3 }); + expect(await readEmbeddingJobRecord(directory, 'job.json')).toEqual({ cursor: 3 }); + const file = path.join(directory, 'job.json'); + await fs.writeFile(file, (await fs.readFile(file, 'utf8')).replace('"cursor":3', '"cursor":4')); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('checksum'); + await expect(writeEmbeddingJobRecord(directory, '../job.json', {})).rejects.toThrow('name'); + await expect( + writeEmbeddingJobRecord(directory, 'job.json', { padding: 'x'.repeat(40_000) }), + ).rejects.toThrow('size limit'); + await fs.writeFile(file, 'x'.repeat(40_000)); + await expect(readEmbeddingJobRecord(directory, 'job.json')).rejects.toThrow('size limit'); + }); +}); diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 92461bbe2..dbfdc9ef5 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -22,6 +22,7 @@ const { executeWithReusedStatementMock, fetchExistingEmbeddingHashesMock, runEmbeddingPipelineMock, + runAnalysisEmbeddingPhaseMock, resolveEmbeddingIdentityMock, installEmbeddingRuntimeMock, resolveEmbeddingRuntimeMock, @@ -40,6 +41,7 @@ const { executeWithReusedStatementMock: vi.fn(), fetchExistingEmbeddingHashesMock: vi.fn(), runEmbeddingPipelineMock: vi.fn(), + runAnalysisEmbeddingPhaseMock: vi.fn(), resolveEmbeddingIdentityMock: vi.fn(), installEmbeddingRuntimeMock: vi.fn(), resolveEmbeddingRuntimeMock: vi.fn(), @@ -74,6 +76,10 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ fetchExistingEmbeddingHashes: (...args: unknown[]) => fetchExistingEmbeddingHashesMock(...args), })); +vi.mock('../../src/core/analysis-embedding-phase.js', () => ({ + runAnalysisEmbeddingPhase: (...args: unknown[]) => runAnalysisEmbeddingPhaseMock(...args), +})); + vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline: (...args: unknown[]) => runEmbeddingPipelineMock(...args), })); @@ -95,6 +101,18 @@ vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ const IDENTITY = { model: 'test-model', dimensions: 768, provider: 'local' } as const; +vi.mock('../../src/core/ingestion/utils/effective-ram.js', async (importOriginal) => ({ + ...(await importOriginal()), + effectiveRamBytes: () => 8 * 1024 ** 3, +})); +vi.mock('node:v8', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getHeapStatistics: () => ({ ...actual.getHeapStatistics(), heap_size_limit: 64 * 1024 ** 3 }), + }; +}); + const BASE_META = { repoPath: '/tmp/emb-sync-repo', lastCommit: 'abc123', @@ -164,6 +182,9 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { loadMetaMock.mockReset().mockResolvedValue({ ...BASE_META }); saveMetaMock.mockReset().mockResolvedValue(undefined); initLbugMock.mockReset().mockResolvedValue(undefined); + runAnalysisEmbeddingPhaseMock + .mockReset() + .mockResolvedValue({ ...BASE_META, stats: { embeddings: 2 } }); closeLbugMock.mockReset().mockResolvedValue(undefined); executeQueryMock.mockReset().mockResolvedValue([{ cnt: 2 }]); executeWithReusedStatementMock.mockReset(); @@ -191,6 +212,16 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); + it('uses the RAM-sized automatic heap despite a larger default V8 limit', async () => { + await store(); + await run(); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + resources: expect.objectContaining({ automatic: true, mainHeapMb: 6553 }), + }), + ); + }); + it('acquires the index lock before re-reading metadata and releases it in finally', async () => { const { dir } = await store(); const order: string[] = []; @@ -205,8 +236,9 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { order.push('loadMeta'); return { ...BASE_META }; }); - initLbugMock.mockImplementation(async () => { - order.push('init'); + runAnalysisEmbeddingPhaseMock.mockImplementation(async () => { + order.push('phase'); + return BASE_META; }); await run(); @@ -214,7 +246,7 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { expect(acquireIndexLockMock).toHaveBeenCalledWith(dir, { sweep: false }); expect(order[0]).toBe('lock'); expect(order.indexOf('loadMeta')).toBeGreaterThan(order.indexOf('lock')); - expect(order.indexOf('init')).toBeGreaterThan(order.indexOf('loadMeta')); + expect(order.indexOf('phase')).toBeGreaterThan(order.indexOf('loadMeta')); expect(order.at(-1)).toBe('release'); }); @@ -406,119 +438,51 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await run(); expect(ensurePrivateSharedGraphMock).toHaveBeenCalledTimes(1); - expect(runEmbeddingPipelineMock).toHaveBeenCalledTimes(1); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledTimes(1); expect(releaseMock).toHaveBeenCalledTimes(1); expect(existsSync(path.join(dir, 'analyze.lock'))).toBe(false); }); - it('supplies the checkout root and indexed source policy, not the storage directory', async () => { + it('does not clear durable partial work when the shared phase fails', async () => { const { dir } = await store(); - const fileHashes = { 'sample.ts': 'a'.repeat(64) }; - loadMetaMock.mockResolvedValue({ ...BASE_META, fileHashes, contentRetention: 'symbol' }); - await run(); - expect(runEmbeddingPipelineMock.mock.calls[0]?.[6].source).toEqual({ - repoPath: BASE_META.repoPath, - fileHashes, - contentRetention: 'symbol', - }); - expect(dir).not.toBe(BASE_META.repoPath); - }); - - it('persists an interrupted checkpoint from the pipeline checkpoint callbacks', async () => { - // The resume contract lives in these callbacks; a mock that never invokes - // them leaves the whole save path unexecuted. - await store(); - runEmbeddingPipelineMock.mockImplementation( - async ( - _executeQuery: unknown, - _executeWithReusedStatement: unknown, - _onProgress: unknown, - _config: unknown, - _signal: unknown, - _existing: unknown, - options: { - onCheckpointWindowStart: (checkpoint: Record) => Promise; - onCheckpoint: (checkpoint: Record) => Promise; - }, - ) => { - await options.onCheckpointWindowStart({ - nodeIds: ['n2', 'n3'], - nodesProcessed: 1, - totalNodes: 3, - chunksProcessed: 1, - }); - await options.onCheckpoint({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 3 }); - return { nodesProcessed: 3, chunksProcessed: 3, failedNodeIds: [] }; - }, + const retained = { ...BASE_META, embeddingJob: { identity: 'retained-work' } }; + loadMetaMock.mockResolvedValue(retained); + runAnalysisEmbeddingPhaseMock.mockRejectedValue(new Error('provider unavailable')); + await expect(run()).rejects.toThrow('provider unavailable'); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + metadataDirectory: dir, + nodeLimit: 0, + }), ); - - await run(); - - type SavedMeta = { - stats?: { embeddings?: number }; - embeddingCheckpoint?: { pendingNodeIds?: string[] }; - }; - const windowStart = saveMetaMock.mock.calls[0]?.[1] as SavedMeta; - expect(windowStart.embeddingCheckpoint?.pendingNodeIds).toEqual(['n2', 'n3']); - // The window marker carries no count, so the last known one must survive. - expect(windowStart.stats?.embeddings).toBe(1); - - const windowEnd = saveMetaMock.mock.calls[1]?.[1] as SavedMeta; - expect(windowEnd.embeddingCheckpoint?.pendingNodeIds).toEqual([]); - expect(windowEnd.stats?.embeddings).toBe(2); + expect(saveMetaMock).not.toHaveBeenCalled(); + expect(runEmbeddingPipelineMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); }); - it('keeps a partial checkpoint when some nodes failed to embed', async () => { + it('keeps the shared phase error without opening or closing a legacy writable graph', async () => { await store(); - runEmbeddingPipelineMock.mockResolvedValue({ - nodesProcessed: 2, - chunksProcessed: 2, - failedNodeIds: ['n9'], - }); - - await run(); - - const saved = saveMetaMock.mock.calls.at(-1)?.[1] as { - embeddingCheckpoint?: { pendingNodeIds?: string[] }; - }; - expect(saved.embeddingCheckpoint).toBeDefined(); - expect(saved.embeddingCheckpoint?.pendingNodeIds).toEqual(['n9']); - }); - - it('does not publish a missing count cell as zero', async () => { - await store(); - executeQueryMock.mockResolvedValue([{}]); - - await expect(run()).rejects.toThrow('Could not verify persisted embedding count.'); - expect(saveMetaMock).toHaveBeenCalledTimes(1); - const saved = saveMetaMock.mock.calls[0]?.[1] as { - stats?: { embeddings?: number }; - embeddingCheckpoint?: { kind?: string; pendingNodeIds?: string[] }; - }; - expect(saved.stats?.embeddings).toBe(1); - expect(saved.embeddingCheckpoint?.kind).toBe('unverified-count'); - expect(saved.embeddingCheckpoint?.pendingNodeIds).toEqual([]); - expect(releaseMock).toHaveBeenCalled(); - }); - - it('keeps the pipeline error when closeLbug also rejects', async () => { - await store(); - runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom')); + runAnalysisEmbeddingPhaseMock.mockRejectedValue(new Error('phase boom')); closeLbugMock.mockRejectedValue(new Error('close boom')); - - await expect(run()).rejects.toThrow('pipeline boom'); - expect(releaseMock).toHaveBeenCalled(); - }); - - it('keeps the pipeline error when sidecar reap also throws', async () => { - await store(); - runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom')); reapEmbeddingSidecarMock.mockImplementation(() => { throw new Error('reap boom'); }); + await expect(run()).rejects.toThrow('phase boom'); + expect(initLbugMock).not.toHaveBeenCalled(); + expect(closeLbugMock).not.toHaveBeenCalled(); + expect(reapEmbeddingSidecarMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); + }); - await expect(run()).rejects.toThrow('pipeline boom'); - expect(releaseMock).toHaveBeenCalled(); + it('preserves a prepared graph receipt before any embedding or staging cleanup', async () => { + const { dir } = await store(); + const receipt = path.join(dir, 'analysis-publication.json'); + await writeFile(receipt, '{ malformed retained receipt'); + await expect(run()).rejects.toThrow(); + expect(await readFile(receipt, 'utf8')).toBe('{ malformed retained receipt'); + expect(runAnalysisEmbeddingPhaseMock).not.toHaveBeenCalled(); + expect(ensurePrivateSharedGraphMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledOnce(); }); it('does not spawn npm on darwin/x64', async () => { @@ -561,22 +525,29 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { it('does not statically import the embedding pipeline', async () => { const { readFileSync } = await import('node:fs'); const src = readFileSync(new URL('../../src/cli/embeddings-sync.ts', import.meta.url), 'utf8'); - expect(src).not.toMatch(/^import .*embedding-pipeline/m); - expect(src).toContain("await import('../core/embeddings/embedding-pipeline.js')"); + expect(src).not.toMatch(/^import .*analysis-embedding-phase/m); + expect(src).toContain("await import('../core/analysis-embedding-phase.js')"); }); - it('loads existing hashes without materializing cached vectors', async () => { - await store(); - const hashes = new Map([ - ['n1', 'h1'], - ['n2', 'h2'], - ]); - fetchExistingEmbeddingHashesMock.mockResolvedValue(hashes); - + it('passes the owned graph generation without materializing hashes or vectors', async () => { + const { dir, metaPath } = await store(); + const metadata = { + ...BASE_META, + graphPath: 'lbug.generation.' + 'a'.repeat(32), + graphGeneration: { id: 'a'.repeat(32) }, + }; + await writeFile(metaPath, JSON.stringify(metadata)); + await writeFile(path.join(dir, metadata.graphPath), 'owned graph'); + loadMetaMock.mockResolvedValue(metadata); await run(); - - expect(fetchExistingEmbeddingHashesMock).toHaveBeenCalledTimes(1); - const existingArg = runEmbeddingPipelineMock.mock.calls[0]?.[5]; - expect(existingArg).toBe(hashes); + expect(runAnalysisEmbeddingPhaseMock).toHaveBeenCalledWith( + expect.objectContaining({ + graphPath: path.join(dir, metadata.graphPath), + expectedGeneration: metadata.graphGeneration.id, + }), + ); + expect(fetchExistingEmbeddingHashesMock).not.toHaveBeenCalled(); + expect(runEmbeddingPipelineMock).not.toHaveBeenCalled(); + expect(saveMetaMock).not.toHaveBeenCalled(); }); }); diff --git a/gitnexus/test/unit/git-utils.test.ts b/gitnexus/test/unit/git-utils.test.ts index b1e00d727..4ea785b4d 100644 --- a/gitnexus/test/unit/git-utils.test.ts +++ b/gitnexus/test/unit/git-utils.test.ts @@ -416,6 +416,45 @@ describe('getCanonicalRepoRoot', () => { }); }); +describe('resolveRepoIdentityRoot', () => { + it.each(['ordinary', 'GIT_DIR', 'GIT_COMMON_DIR', 'commondir', 'symlink'] as const)( + 'preserves Git identity for %s checkouts', + async (layout) => { + const { getCanonicalRepoRoot, resolveRepoIdentityRoot } = + await import('../../src/storage/git.js'); + const root = fs.realpathSync.native(makeIsolatedTempDir('gitnexus-identity-')); + let repo = path.join(root, 'repo'); + const other = path.join(root, 'other'); + vi.stubEnv('GIT_DIR', undefined); + vi.stubEnv('GIT_COMMON_DIR', undefined); + try { + for (const directory of [repo, other]) { + fs.mkdirSync(directory); + execFileSync(gitExecutable, ['init', '-q'], { cwd: directory, stdio: 'pipe' }); + } + if (layout === 'GIT_DIR' || layout === 'GIT_COMMON_DIR') + vi.stubEnv(layout, path.join(other, '.git')); + else if (layout === 'commondir') + fs.writeFileSync(path.join(repo, '.git', 'commondir'), path.join(other, '.git') + '\n'); + else if (layout === 'symlink') { + const link = path.join(root, 'linked-name'); + fs.symlinkSync(repo, link, 'junction'); + repo = link; + } + const canonical = getCanonicalRepoRoot(repo); + expect(canonical).not.toBeNull(); + expect(resolveRepoIdentityRoot(repo)).toBe(canonical); + const subdir = path.join(repo, 'subdir'); + fs.mkdirSync(subdir); + expect(resolveRepoIdentityRoot(subdir)).toBe(path.resolve(subdir)); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); + // ─── selfCommitContextFiles (#2639) ──────────────────────────────────────── describe('selfCommitContextFiles', () => { diff --git a/gitnexus/test/unit/hooks-shared-store.test.ts b/gitnexus/test/unit/hooks-shared-store.test.ts index 074767d3d..e9c65faef 100644 --- a/gitnexus/test/unit/hooks-shared-store.test.ts +++ b/gitnexus/test/unit/hooks-shared-store.test.ts @@ -5,6 +5,7 @@ import path from 'path'; import { createRequire } from 'module'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { slotNameForCanonicalPath } from '../../src/storage/storage-slot.js'; +import { branchSlug } from '../../src/storage/branch-index.js'; /** * #3352 — the Claude hook resolves a shared-store checkout to the commit @@ -102,6 +103,61 @@ describe('registry-query shared store graph (#3352)', () => { for (const copy of copies) expect(copy).toBe(primary); }); + it.each(HOOK_COPIES)('resolves a private generation in a registered branch slot (%s)', (file) => { + const branch = 'feature/private'; + execFileSync( + 'git', + [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.invalid', + 'commit', + '--allow-empty', + '-m', + 'fixture', + ], + { cwd: checkout, stdio: 'ignore' }, + ); + execFileSync('git', ['checkout', '-b', branch], { cwd: checkout, stdio: 'ignore' }); + writeSlot({ graphPath: commitGraph }); + fs.writeFileSync( + path.join(home, 'registry.json'), + JSON.stringify([ + { + name: 'wt', + path: checkout, + storagePath: slot, + indexedAt: '', + lastCommit: '', + branches: [{ branch }], + }, + ]), + ); + const indexDir = path.join(slot, 'branches', branchSlug(branch)); + const graph = `lbug.generation.${'a'.repeat(32)}`; + fs.mkdirSync(indexDir, { recursive: true }); + fs.writeFileSync( + path.join(indexDir, 'gitnexus.json'), + JSON.stringify({ repoPath: checkout, graphPath: graph }), + ); + fs.writeFileSync(path.join(indexDir, graph), 'graph'); + expect(load(file).findRegisteredRepo(checkout)?.lbugPath).toBe(path.join(indexDir, graph)); + }); + + it.each(HOOK_COPIES)('resolves an unregistered local private generation (%s)', (file) => { + const indexDir = path.join(checkout, '.gitnexus'); + const graph = `lbug.generation.${'a'.repeat(32)}`; + fs.mkdirSync(indexDir, { recursive: true }); + fs.writeFileSync( + path.join(indexDir, 'gitnexus.json'), + JSON.stringify({ repoPath: checkout, graphPath: graph }), + ); + fs.writeFileSync(path.join(indexDir, graph), 'graph'); + const hook = load(file) as unknown as { findLocalOwnedRepo: (cwd: string) => HookRepo }; + expect(hook.findLocalOwnedRepo(checkout)?.lbugPath).toBe(path.join(indexDir, graph)); + }); + it.each(HOOK_COPIES)('returns the commit graph a shared slot records (%s)', (file) => { writeSlot({ graphPath: commitGraph }); expect(load(file).findRegisteredRepo(checkout)?.lbugPath).toBe(commitGraph); diff --git a/gitnexus/test/unit/http-embedding-response-budget.test.ts b/gitnexus/test/unit/http-embedding-response-budget.test.ts new file mode 100644 index 000000000..5d2d6a994 --- /dev/null +++ b/gitnexus/test/unit/http-embedding-response-budget.test.ts @@ -0,0 +1,123 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.resetModules(); +}); + +async function client() { + vi.stubEnv('GITNEXUS_EMBEDDING_URL', 'https://budget.example/v1'); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'test-model'); + vi.stubEnv('GITNEXUS_EMBEDDING_DIMS', '4'); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '1'); + return (await import('../../src/core/embeddings/http-client.js')).httpEmbed; +} + +describe('bounded HTTP embedding responses', () => { + it('preserves vectors when the JSON arrives across one-byte UTF-8 chunks', async () => { + const embed = await client(); + const body = Buffer.from( + JSON.stringify({ data: [{ embedding: [0.25, 0.5, 0.75, 1] }], model: '漢😀' }), + ); + vi.stubGlobal( + 'fetch', + async () => + new Response( + new ReadableStream({ + start(controller) { + for (const byte of body) controller.enqueue(new Uint8Array([byte])); + controller.close(); + }, + }), + ), + ); + const result = await embed(['text'], { maxResponseBytes: body.byteLength }); + expect(Array.from(result[0])).toEqual([0.25, 0.5, 0.75, 1]); + }); + + it.each([undefined, '1'])( + 'cancels an oversized stream with content-length %s', + async (length) => { + const embed = await client(); + const cancel = vi.fn(); + let pulls = 0; + vi.stubGlobal( + 'fetch', + async () => + new Response( + new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array(32)); + }, + cancel, + }), + { headers: length === undefined ? {} : { 'content-length': length } }, + ), + ); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(cancel).toHaveBeenCalledOnce(); + expect(pulls).toBeLessThanOrEqual(4); + }, + ); + + it('refuses an oversized declared body before reading or decoding it', async () => { + const embed = await client(); + const cancel = vi.fn(); + vi.stubGlobal( + 'fetch', + async () => + new Response(new ReadableStream({ cancel }), { + headers: { 'content-length': '1000000000' }, + }), + ); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(cancel).toHaveBeenCalledOnce(); + }); + + it('does not retry a deterministic response-budget violation', async () => { + const embed = await client(); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '3'); + const fetch = vi.fn( + async () => new Response('x'.repeat(100), { headers: { 'content-length': '100' } }), + ); + vi.stubGlobal('fetch', fetch); + await expect(embed(['text'], { maxResponseBytes: 64 })).rejects.toThrow( + '64-byte response budget', + ); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it('retains the existing body retry behavior under the byte limit', async () => { + const embed = await client(); + vi.stubEnv('GITNEXUS_EMBEDDING_MAX_ATTEMPTS', '2'); + vi.stubEnv('GITNEXUS_EMBEDDING_RETRY_CAP_MS', '1'); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response('{"data":')) + .mockResolvedValueOnce(new Response(JSON.stringify({ data: [{ embedding: [1, 0, 0, 0] }] }))); + vi.stubGlobal('fetch', fetch); + expect(Array.from((await embed(['text'], { maxResponseBytes: 1024 }))[0])).toEqual([ + 1, 0, 0, 0, + ]); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it.each([0, -1, NaN, Infinity, 1.5])( + 'rejects invalid budget %s before fetching', + async (maximum) => { + const embed = await client(); + const fetch = vi.fn(); + vi.stubGlobal('fetch', fetch); + await expect(embed(['text'], { maxResponseBytes: maximum })).rejects.toThrow( + 'Invalid embedding response', + ); + expect(fetch).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts index d2c32a280..99d53b499 100644 --- a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts +++ b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts @@ -17,11 +17,44 @@ import { describe, it, expect } from 'vitest'; import { detectGraphWriteCollapse, getIndexIncompleteReasons, + getGraphIncompleteReasons, GRAPH_WRITE_COLLAPSE_MIN_EDGES, GRAPH_WRITE_COLLAPSE_RATIO, INDEX_INCOMPLETE_REASONS, } from '../../src/core/index-freshness.js'; +it('keeps graph failures visible while separating legacy and durable embedding debt', () => { + const metadata = { + scopeExtractionReceipt: 1 as const, + embeddingJob: { + version: 1 as const, + identity: 'a'.repeat(64), + graphGeneration: 'b'.repeat(32), + }, + graphWriteCollapsed: { expected: 100, persisted: 0 }, + }; + expect(getIndexIncompleteReasons(metadata)).toEqual([ + 'graph-write-collapsed', + 'embedding-job-pending', + ]); + expect(getGraphIncompleteReasons(metadata)).toEqual(['graph-write-collapsed']); + expect( + getGraphIncompleteReasons({ + scopeExtractionReceipt: 1, + embeddingCheckpoint: { + at: '', + model: 'fixture', + dimensions: 3, + provider: 'local', + nodesProcessed: 0, + totalNodes: 1, + chunksProcessed: 0, + }, + }), + ).toEqual([]); + expect(getGraphIncompleteReasons(null)).toEqual(['scope-extraction-unverified']); +}); + describe('detectGraphWriteCollapse (B2 detection)', () => { it('flags the reported field failure (23009 built, 2170 persisted)', () => { expect(detectGraphWriteCollapse(23009, 2170)).toEqual({ diff --git a/gitnexus/test/unit/index-lock.test.ts b/gitnexus/test/unit/index-lock.test.ts index 120e09727..8e46b2b0d 100644 --- a/gitnexus/test/unit/index-lock.test.ts +++ b/gitnexus/test/unit/index-lock.test.ts @@ -179,6 +179,18 @@ describe('release', () => { }); describe('sweepStagingArtifacts', () => { + it('reclaims private generation stages and WALs while retaining published graphs', () => { + const generation = `lbug.generation.${'a'.repeat(32)}`; + const stage = `${generation}.staging.6e34c761-bf58-46cc-8b54-78d11607bc46`; + for (const name of [generation, `${generation}.wal`, stage, `${stage}.wal`]) + writeFileSync(path.join(dir, name), 'graph'); + sweepStagingArtifacts(dir); + expect(existsSync(path.join(dir, generation))).toBe(true); + expect(existsSync(path.join(dir, `${generation}.wal`))).toBe(true); + expect(existsSync(path.join(dir, stage))).toBe(false); + expect(existsSync(path.join(dir, `${stage}.wal`))).toBe(false); + }); + const recoveryStage = 'lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46'; const seedRecovery = (stagingFile = recoveryStage): void => { writeFileSync( diff --git a/gitnexus/test/unit/lbug-analysis-resources.test.ts b/gitnexus/test/unit/lbug-analysis-resources.test.ts new file mode 100644 index 000000000..c27fff29e --- /dev/null +++ b/gitnexus/test/unit/lbug-analysis-resources.test.ts @@ -0,0 +1,262 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const native = vi.hoisted(() => ({ + open: vi.fn(), + close: vi.fn(async () => {}), + quarantine: vi.fn(async () => ({ quarantinedPath: '/unused' })), +})); +vi.mock('../../src/core/lbug/lbug-config.js', async (original) => ({ + ...(await original()), + openLbugConnection: native.open, + closeLbugConnection: native.close, +})); +vi.mock('../../src/core/lbug/sidecar-recovery.js', async (original) => ({ + ...(await original()), + quarantineWalForMissingShadow: native.quarantine, + isMissingShadowSidecarError: (error: unknown) => + error instanceof Error && error.message === 'missing shadow test', +})); +vi.mock('../../src/core/lbug/extension-loader.js', async (original) => ({ + ...(await original()), + extensionManager: { + ensure: vi.fn(async () => true), + reset: vi.fn(), + getCapabilities: vi.fn(() => []), + }, +})); + +describe('analysis database resource allocations', () => { + let root: string; + let adapter: typeof import('../../src/core/lbug/lbug-adapter.js'); + beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-analysis-resources-')); + native.open.mockImplementation(async () => ({ + db: { close: vi.fn(async () => {}) }, + conn: { + query: vi.fn(async () => ({ getAll: async () => [], close: () => {} })), + close: vi.fn(async () => {}), + }, + })); + adapter = await import('../../src/core/lbug/lbug-adapter.js'); + }); + afterEach(async () => { + await adapter.closeLbug(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('applies native allocations and reopens only when the allocation changes', async () => { + const dbPath = path.join(root, 'lbug'); + const options = { skipFts: true, bufferPoolBytes: 64 * 1024 * 1024, threads: 3 }; + await adapter.initLbug(dbPath, options); + await adapter.initLbug(dbPath, { ...options }); + expect(native.open).toHaveBeenCalledTimes(1); + expect(native.open.mock.calls[0][2]).toEqual({ + bufferPoolBytes: options.bufferPoolBytes, + threads: 3, + }); + const first = await native.open.mock.results[0].value; + await adapter.initLbug(dbPath, { ...options, threads: 2 }); + expect(first.conn.close).toHaveBeenCalledTimes(1); + expect(first.db.close).toHaveBeenCalledTimes(1); + expect(native.open).toHaveBeenCalledTimes(2); + expect(native.open.mock.calls[1][2]).toEqual({ + bufferPoolBytes: options.bufferPoolBytes, + threads: 2, + }); + }); + + it('rejects invalid allocations before creating the database directory or opening native code', async () => { + const parent = path.join(root, 'not-created'); + for (const options of [ + { bufferPoolBytes: 0 }, + { bufferPoolBytes: 1024 }, + { threads: 0 }, + { threads: 1.5 }, + ]) { + await expect(adapter.initLbug(path.join(parent, 'lbug'), options)).rejects.toThrow( + /Explicit LadybugDB/, + ); + } + expect(native.open).not.toHaveBeenCalled(); + await expect(fs.stat(parent)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.each(['conn', 'db'] as const)( + 'retains a failed %s close and refuses to allocate another pool', + async (handle) => { + const dbPath = path.join(root, 'lbug'); + const options = { skipFts: true, bufferPoolBytes: 64 * 1024 * 1024, threads: 3 }; + await adapter.initLbug(dbPath, options); + const first = await native.open.mock.results[0].value; + first[handle].close.mockRejectedValueOnce(new Error('native close failed')); + await expect(adapter.initLbug(dbPath, { ...options, threads: 2 })).rejects.toThrow( + 'native close failed', + ); + expect(native.open).toHaveBeenCalledTimes(1); + // A subsequent successful drain can safely admit the requested allocation. + await adapter.initLbug(dbPath, options); + expect(native.open).toHaveBeenCalledTimes(2); + }, + ); + + it('keeps the same allocations when missing-shadow recovery reopens the writer', async () => { + native.open.mockResolvedValueOnce({ + db: { close: vi.fn(async () => {}) }, + conn: { + query: vi.fn(async () => { + throw new Error('missing shadow test'); + }), + close: vi.fn(async () => {}), + }, + }); + const options = { skipFts: true, bufferPoolBytes: 32 * 1024 * 1024, threads: 2 }; + await adapter.initLbug(path.join(root, 'lbug'), options); + expect(native.quarantine).toHaveBeenCalledTimes(1); + expect(native.open).toHaveBeenCalledTimes(2); + expect(native.open.mock.calls.map((call) => call[2])).toEqual([ + { bufferPoolBytes: options.bufferPoolBytes, threads: 2 }, + { bufferPoolBytes: options.bufferPoolBytes, threads: 2 }, + ]); + }); + + it.each(['parser', 'embedding'] as const)( + 'refuses %s admission while the main pool cannot close', + async (phase) => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const { runPublishedEmbeddingJob } = + await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + await adapter.initLbug(path.join(root, 'lbug'), { + skipFts: true, + bufferPoolBytes: 32 * 1024 * 1024, + threads: 1, + }); + const first = await native.open.mock.results[0].value; + first.db.close.mockRejectedValue(new Error('main pool still allocated')); + await expect(adapter.closeLbug()).rejects.toThrow('main pool still allocated'); + const loadMetadata = vi.fn(async () => null); + const provider = vi.fn(); + const onProgress = vi.fn(); + try { + const run = + phase === 'parser' + ? runFullAnalysis(root, { resources }, { onProgress }) + : runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, 'lbug'), + resources: resources.embedding, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ); + await expect(run).rejects.toThrow('main pool still allocated'); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + expect(onProgress).not.toHaveBeenCalled(); + expect(native.open).toHaveBeenCalledTimes(1); + } finally { + first.db.close.mockResolvedValue(); + } + }, + ); + + it('waits for an active session before releasing its native handle', async () => { + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const wait = new Promise((resolve) => { + release = resolve; + }); + const operation = adapter.withLbugDb( + path.join(root, 'lbug'), + async () => { + started(); + await wait; + }, + { skipFts: true }, + ); + await ready; + const first = await native.open.mock.results[0].value; + let drained = false; + const drain = adapter.drainAnalysisNativeResources().then(() => { + drained = true; + }); + try { + await Promise.resolve(); + expect(drained).toBe(false); + expect(first.conn.close).not.toHaveBeenCalled(); + expect(first.db.close).not.toHaveBeenCalled(); + } finally { + release(); + await Promise.all([operation, drain]); + } + expect(first.conn.close).toHaveBeenCalledTimes(1); + expect(first.db.close).toHaveBeenCalledTimes(1); + }); + + it('does not reopen native storage when an idle phase has nothing to release', async () => { + await adapter.drainAnalysisNativeResources(); + expect(native.open).not.toHaveBeenCalled(); + }); + + it('honors cancellation during admission before reading job metadata', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runPublishedEmbeddingJob } = await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + let started!: () => void; + let release!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const wait = new Promise((resolve) => { + release = resolve; + }); + const operation = adapter.withLbugDb( + path.join(root, 'lbug'), + async () => { + started(); + await wait; + }, + { skipFts: true }, + ); + await ready; + const cancellation = new AbortController(); + const loadMetadata = vi.fn(async () => null); + const provider = vi.fn(); + const outcome = runPublishedEmbeddingJob( + { + metadataDirectory: root, + graphPath: path.join(root, 'lbug'), + resources: resources.embedding, + signal: cancellation.signal, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ).catch((error: unknown) => error); + cancellation.abort(); + release(); + await operation; + expect(await outcome).toMatchObject({ name: 'AbortError' }); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/lbug-checkpoint.test.ts b/gitnexus/test/unit/lbug-checkpoint.test.ts index f898030e2..d13a385c3 100644 --- a/gitnexus/test/unit/lbug-checkpoint.test.ts +++ b/gitnexus/test/unit/lbug-checkpoint.test.ts @@ -56,7 +56,7 @@ describe('flushWAL / safeClose — consolidation guard (#1376)', () => { it('closeLbug delegates to safeClose instead of inlining conn.close/db.close', () => { // Match `closeLbug =` precisely so we don't prefix-match `closeLbugBeforeExit`. const closeLbugBody = adapterSource.slice(adapterSource.indexOf('export const closeLbug =')); - expect(closeLbugBody).toMatch(/await safeClose\(\)/); + expect(closeLbugBody).toMatch(/await safeClose\(options\)/); // closeLbug must NOT contain its own conn.close() or db.close() — those // live exclusively inside safeClose now. const closeLbugBlock = closeLbugBody.slice(0, closeLbugBody.indexOf('export const', 1) >>> 0); diff --git a/gitnexus/test/unit/lbug-explicit-budget.test.ts b/gitnexus/test/unit/lbug-explicit-budget.test.ts new file mode 100644 index 000000000..5397e5ff2 --- /dev/null +++ b/gitnexus/test/unit/lbug-explicit-budget.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createLbugDatabase, openLbugConnection } from '../../src/core/lbug/lbug-config.js'; + +afterEach(() => vi.unstubAllEnvs()); + +describe('explicit native handle reservations', () => { + it('uses a per-handle reservation while preserving the environment policy for legacy callers', () => { + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', '0'); + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database } as unknown as Parameters[0]; + createLbugDatabase(module, '/tmp/budgeted', { bufferPoolBytes: 64 * 1024 * 1024 }); + expect(Database.mock.calls[0][1]).toBe(64 * 1024 * 1024); + createLbugDatabase(module, '/tmp/legacy'); + expect(Database.mock.calls[1][1]).toBe(0); + }); + + it.each([0, -1, NaN, Infinity, 1.5])( + 'rejects invalid explicit reservation %s before opening native', + (bufferPoolBytes) => { + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database } as unknown as Parameters[0]; + expect(() => createLbugDatabase(module, '/tmp/invalid', { bufferPoolBytes })).toThrow( + 'positive byte', + ); + expect(Database).not.toHaveBeenCalled(); + }, + ); + + it('passes an explicit thread count only when requested', async () => { + const Database = vi.fn<(...args: unknown[]) => void>(function () {}); + const Connection = vi.fn<(...args: unknown[]) => void>(function () {}); + const module = { Database, Connection } as unknown as Parameters[0]; + await openLbugConnection(module, '/tmp/threads', { + bufferPoolBytes: 64 * 1024 * 1024, + threads: 2, + }); + expect(Connection.mock.calls[0]).toHaveLength(2); + expect(Connection.mock.calls[0][1]).toBe(2); + await openLbugConnection(module, '/tmp/legacy'); + expect(Connection.mock.calls[1]).toHaveLength(1); + await expect(openLbugConnection(module, '/tmp/invalid', { threads: 0 })).rejects.toThrow( + 'positive integer', + ); + expect(Database).toHaveBeenCalledTimes(2); + }); +}); diff --git a/gitnexus/test/unit/lbug-resource-release.test.ts b/gitnexus/test/unit/lbug-resource-release.test.ts new file mode 100644 index 000000000..094e3ed96 --- /dev/null +++ b/gitnexus/test/unit/lbug-resource-release.test.ts @@ -0,0 +1,184 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type lbug from '@ladybugdb/core'; +import type { LbugConnectionHandle } from '../../src/core/lbug/lbug-config.js'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; + +type LbugModule = typeof lbug; + +describe('reserved native resource release', () => { + let native: typeof import('../../src/core/lbug/lbug-config.js'); + beforeEach(async () => { + vi.resetModules(); + native = await import('../../src/core/lbug/lbug-config.js'); + }); + + function handles() { + const conn = { close: vi.fn(async () => {}) }; + const db = { close: vi.fn(async () => {}) }; + return { conn, db, handle: { conn, db } as unknown as LbugConnectionHandle }; + } + + it.each(['conn', 'db'] as const)( + 'propagates a failed %s close and retries its release', + async (part) => { + const h = handles(); + h[part].close.mockRejectedValueOnce(new Error('release failed')); + await expect(native.closeLbugConnection(h.handle, { strict: true })).rejects.toThrow( + 'release failed', + ); + expect(h.db.close).toHaveBeenCalledTimes(part === 'conn' ? 0 : 1); + await native.closeLbugConnection(h.handle, { strict: true }); + expect(h[part].close).toHaveBeenCalledTimes(2); + expect(h[part === 'conn' ? 'db' : 'conn'].close).toHaveBeenCalledTimes(1); + }, + ); + + it('coalesces concurrent closes and waits for the physical release', async () => { + const h = handles(); + let release!: () => void; + h.conn.close.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + let completed = false; + const first = native.closeLbugConnection(h.handle, { strict: true }); + const second = native.closeLbugConnection(h.handle, { strict: true }).then(() => { + completed = true; + }); + await Promise.resolve(); + expect(completed).toBe(false); + expect(h.conn.close).toHaveBeenCalledTimes(1); + expect(h.db.close).not.toHaveBeenCalled(); + release(); + await Promise.all([first, second]); + expect(h.db.close).toHaveBeenCalledTimes(1); + }); + + it('blocks another reserved pool until a failed release succeeds', async () => { + const h = handles(); + h.db.close.mockRejectedValue(new Error('pool still allocated')); + await native.closeLbugConnection(h.handle, { strict: true }).catch(() => {}); + const Database = vi.fn(function () { + return handles().db; + }); + const Connection = vi.fn(function () { + return handles().conn; + }); + const module = { Database, Connection } as unknown as LbugModule; + await expect( + native.openLbugConnection(module, '/unused', { bufferPoolBytes: 8 * 1024 * 1024 }), + ).rejects.toThrow('pool still allocated'); + expect(Database).not.toHaveBeenCalled(); + h.db.close.mockResolvedValue(); + const next = await native.openLbugConnection(module, '/unused', { + bufferPoolBytes: 8 * 1024 * 1024, + }); + expect(Database).toHaveBeenCalledTimes(1); + await native.closeLbugConnection(next); + }); + + it('retains a database whose connection constructor and cleanup both failed', async () => { + const h = handles(); + h.db.close.mockRejectedValue(new Error('orphan pool still allocated')); + const Database = vi.fn(function () { + return h.db; + }); + const Connection = vi.fn(function (): typeof h.conn { + throw new Error('connection construction failed'); + }); + const module = { Database, Connection } as unknown as LbugModule; + const allocation = { bufferPoolBytes: 8 * 1024 * 1024 }; + await expect(native.openLbugConnection(module, '/unused', allocation)).rejects.toThrow( + /connection construction|release/, + ); + await expect(native.openLbugConnection(module, '/unused', allocation)).rejects.toThrow( + 'orphan pool still allocated', + ); + expect(Database).toHaveBeenCalledTimes(1); + h.db.close.mockResolvedValue(); + Connection.mockImplementation(function () { + return h.conn; + }); + const next = await native.openLbugConnection(module, '/unused', allocation); + expect(Database).toHaveBeenCalledTimes(2); + await native.closeLbugConnection(next); + }); + + it('preserves best-effort close for callers without a reservation', async () => { + const h = handles(); + h.conn.close.mockRejectedValue(new Error('legacy close failure')); + await native.closeLbugConnection(h.handle); + expect(h.db.close).toHaveBeenCalledTimes(1); + }); + + it('keeps strict release mandatory for a handle opened with a reservation', async () => { + const h = handles(); + const Database = vi.fn(function () { + return h.db; + }); + const Connection = vi.fn(function () { + return h.conn; + }); + const handle = await native.openLbugConnection( + { Database, Connection } as unknown as LbugModule, + '/unused', + { threads: 1, bufferPoolBytes: 8 * 1024 * 1024 }, + ); + h.db.close.mockRejectedValueOnce(new Error('reserved close failed')); + await expect(native.closeLbugConnection(handle)).rejects.toThrow('reserved close failed'); + await native.closeLbugConnection(handle); + expect(h.conn.close).toHaveBeenCalledTimes(1); + expect(h.db.close).toHaveBeenCalledTimes(2); + }); + + it('blocks parser and model admission while a prior native release is still failing', async () => { + const { resolveAnalysisResourceBudget } = + await import('../../src/core/analysis-resource-budget.js'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const { runPublishedEmbeddingJob } = await import('../../src/core/embeddings/embedding-job.js'); + const resources = resolveAnalysisResourceBudget({ + effectiveMemoryBytes: 8 * 1024 ** 3, + availableCpus: 4, + modelMemoryMb: 0, + }); + const h = handles(); + h.db.close.mockRejectedValue(new Error('previous allocation is live')); + await native.closeLbugConnection(h.handle, { strict: true }).catch(() => {}); + const onProgress = vi.fn(); + const loadMetadata = vi.fn(); + const provider = vi.fn(); + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-release-admission-')); + execFileSync('git', ['init', '--quiet', repo]); + vi.stubEnv('GITNEXUS_HOME', path.join(repo, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + try { + await expect(runFullAnalysis(repo, { resources }, { onProgress })).rejects.toThrow( + 'previous allocation is live', + ); + await expect( + runPublishedEmbeddingJob( + { + metadataDirectory: path.join(repo, '.gitnexus'), + graphPath: path.join(repo, '.gitnexus', 'lbug'), + resources: resources.embedding, + metadata: { loadMetadata, saveMetadata: vi.fn() }, + }, + { provider }, + ), + ).rejects.toThrow('previous allocation is live'); + expect(onProgress).not.toHaveBeenCalled(); + expect(loadMetadata).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + } finally { + h.db.close.mockResolvedValue(); + await native.drainLbugResourceReleases(); + vi.unstubAllEnvs(); + await fs.rm(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/lbug-vector-pool.test.ts b/gitnexus/test/unit/lbug-vector-pool.test.ts new file mode 100644 index 000000000..1f1bc6a26 --- /dev/null +++ b/gitnexus/test/unit/lbug-vector-pool.test.ts @@ -0,0 +1,191 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + fts: vi.fn(), + vector: vi.fn(), + threads: vi.fn(), + connections: [] as Array<{ close: ReturnType }>, +})); +vi.mock('@ladybugdb/core', () => ({ + default: { + Connection: class { + close = vi.fn(async () => {}); + setMaxNumThreadForExec = mocks.threads; + constructor() { + mocks.connections.push(this); + } + async query() { + return { getAll: async () => [], close: async () => {} }; + } + }, + }, +})); +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + isReadOnlyDbError: () => false, + loadFTSExtension: mocks.fts, + loadVectorExtension: mocks.vector, +})); +vi.mock('../../src/core/lbug/lbug-config.js', async (original) => ({ + ...(await original()), + createLbugDatabase: mocks.create, +})); +import { + closeLbug, + initLbug, + isLbugReady, + ensureVectorExtension, +} from '../../src/core/lbug/pool-adapter.js'; + +let directory: string; +const options = { kind: 'vector' as const, bufferPoolBytes: 64 * 1024 * 1024, threads: 2 }; +async function file(name: string) { + const target = path.join(directory, name); + await fs.writeFile(target, 'fixture'); + return target; +} +beforeEach(async () => { + vi.clearAllMocks(); + mocks.connections.length = 0; + mocks.create.mockImplementation(() => ({ + init: vi.fn(async () => {}), + close: vi.fn(async () => {}), + })); + mocks.fts.mockResolvedValue(true); + mocks.vector.mockResolvedValue(false); + mocks.threads.mockResolvedValue(undefined); + directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-pool-')); +}); +afterEach(async () => { + await closeLbug(); + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('vector pool isolation', () => { + it('shares one native reservation across hard links until the last reader closes', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledOnce(); + await ensureVectorExtension('first'); + await ensureVectorExtension('second'); + expect(mocks.vector).toHaveBeenCalledOnce(); + const database = mocks.create.mock.results[0].value; + await closeLbug('first'); + expect(database.close).not.toHaveBeenCalled(); + expect(isLbugReady('second')).toBe(true); + await closeLbug('second'); + expect(database.close).toHaveBeenCalledOnce(); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + + it('keeps independent files separate even when their bytes match', async () => { + await initLbug('first', await file('first'), options); + await initLbug('second', await file('second'), options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + + it('reopens a replaced vector path without closing a sibling on the previous inode', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await initLbug('second', second, options); + const original = mocks.create.mock.results[0].value; + await fs.rename(await file('replacement'), second); + expect(await initLbug('second', second, options)).toBe(true); + expect(mocks.create).toHaveBeenCalledTimes(2); + expect(original.close).not.toHaveBeenCalled(); + await closeLbug('first'); + expect(original.close).toHaveBeenCalledOnce(); + expect(isLbugReady('second')).toBe(true); + }); + + it('rejects a different reservation through another link to the same vectors', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await expect( + initLbug('second', second, { ...options, bufferPoolBytes: options.bufferPoolBytes * 2 }), + ).rejects.toThrow('different resource reservation'); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + + it('retains a shared native handle when another reader fails to initialize', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + const database = mocks.create.mock.results[0].value; + mocks.threads.mockRejectedValueOnce(new Error('thread configuration failed')); + await expect(initLbug('second', second, options)).rejects.toThrow( + 'thread configuration failed', + ); + expect(database.close).not.toHaveBeenCalled(); + expect(isLbugReady('first')).toBe(true); + await initLbug('second', second, options); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + + it('validates a new link before reusing an already open native handle', async () => { + const first = await file('first'); + const second = path.join(directory, 'second'); + await fs.link(first, second); + await initLbug('first', first, options); + await fs.writeFile(second + '.wal', 'unfinished'); + await expect(initLbug('second', second, options)).rejects.toThrow('unfinished native writes'); + expect(mocks.create).toHaveBeenCalledOnce(); + expect(isLbugReady('first')).toBe(true); + expect(await fs.readFile(second + '.wal', 'utf8')).toBe('unfinished'); + }); + + it('retains the graph-cache capacity while bounding vector residents independently', async () => { + for (let i = 0; i < 5; i++) await initLbug(`graph-${i}`, await file(`graph-${i}`)); + for (let i = 0; i < 6; i++) await initLbug(`vector-${i}`, await file(`vector-${i}`), options); + for (let i = 0; i < 5; i++) expect(isLbugReady(`graph-${i}`)).toBe(true); + expect(isLbugReady('vector-0')).toBe(false); + for (let i = 1; i < 6; i++) expect(isLbugReady(`vector-${i}`)).toBe(true); + expect(mocks.fts).toHaveBeenCalledTimes(5); + expect(mocks.threads).toHaveBeenCalledTimes(6 * 8); + expect(mocks.create).toHaveBeenLastCalledWith( + expect.anything(), + path.join(directory, 'vector-5'), + { readOnly: true, bufferPoolBytes: options.bufferPoolBytes }, + ); + }); + it('retains the actual reservation when its caller mutates an options object', async () => { + const target = await file('vectors'); + const reservation = { ...options }; + await initLbug('vectors', target, reservation); + reservation.bufferPoolBytes *= 2; + await expect(initLbug('vectors', target, reservation)).rejects.toThrow( + 'different resource reservation', + ); + expect(mocks.create).toHaveBeenCalledOnce(); + }); + it('closes an incompletely initialized native handle and permits a clean retry', async () => { + const target = await file('vectors'); + mocks.threads.mockRejectedValueOnce(new Error('thread configuration failed')); + await expect(initLbug('vectors', target, options)).rejects.toThrow( + 'thread configuration failed', + ); + expect(mocks.create.mock.results[0].value.close).toHaveBeenCalledOnce(); + expect(mocks.connections[0].close).toHaveBeenCalledOnce(); + await initLbug('vectors', target, options); + expect(mocks.create).toHaveBeenCalledTimes(2); + }); + it('refuses uncheckpointed vector files without quarantining their sidecars', async () => { + const target = await file('vectors'); + await fs.writeFile(target + '.wal', 'pending writes'); + await expect(initLbug('vectors', target, options)).rejects.toThrow('unfinished native writes'); + expect(mocks.create).not.toHaveBeenCalled(); + expect(await fs.readFile(target + '.wal', 'utf8')).toBe('pending writes'); + }); +}); diff --git a/gitnexus/test/unit/legacy-embedding-source.test.ts b/gitnexus/test/unit/legacy-embedding-source.test.ts new file mode 100644 index 000000000..a427ce605 --- /dev/null +++ b/gitnexus/test/unit/legacy-embedding-source.test.ts @@ -0,0 +1,219 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + recordLegacyEmbeddingSource, + readLegacyEmbeddingSource, +} from '../../src/storage/legacy-embedding-source.js'; +import { + adoptEmbeddingArtifact, + readEmbeddingArtifactSpace, +} from '../../src/storage/embedding-artifact.js'; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-legacy-receipt-')); + roots.push(root); + const sourceFile = 'lbug.generation.' + 'a'.repeat(32); + await fs.writeFile(path.join(root, sourceFile), 'retired database'); + const input = { + sourceFile, + schemaFingerprint: 'schema', + recordedIdentity: { provider: 'local', model: 'old', dimensions: 4 }, + }; + return { root, input, source: path.join(root, sourceFile) }; +} +const collect = async (ids: AsyncIterable) => { + const result: string[] = []; + for await (const id of ids) result.push(id); + return result; +}; + +describe('retained legacy source receipts', () => { + it('retains the sole legacy fallback through rebases and drops it only after sealing', async () => { + const f = await fixture(); + const legacy = await recordLegacyEmbeddingSource(f.root, f.input, []); + const space = { + vectorIdentity: 'a'.repeat(64), + inputIdentity: 'b'.repeat(64), + dimensions: 4, + schema: 'vectors', + }; + const owner = { + artifactIdentity: 'c'.repeat(64), + jobIdentity: 'c'.repeat(64), + sourceIdentity: 'd'.repeat(64), + graphGeneration: 'e'.repeat(32), + manifestDigest: 'f'.repeat(64), + phase: 'pending' as const, + revalidateInputs: false, + fallbackLegacy: legacy, + }; + const validate = async () => {}; + const first = await adoptEmbeddingArtifact(f.root, space, null, owner, validate); + const rebased = { + ...owner, + jobIdentity: '1'.repeat(64), + graphGeneration: '2'.repeat(32), + revalidateInputs: true, + }; + await expect( + adoptEmbeddingArtifact( + f.root, + space, + first, + { ...rebased, fallbackLegacy: undefined }, + validate, + ), + ).rejects.toThrow('retained fallback'); + await expect( + adoptEmbeddingArtifact( + f.root, + space, + first, + { ...owner, fallbackArtifact: '3'.repeat(64) }, + validate, + ), + ).rejects.toThrow('ownership'); + const next = await adoptEmbeddingArtifact(f.root, space, first, rebased, validate); + expect((await readEmbeddingArtifactSpace(f.root, space, validate))?.owner.fallbackLegacy).toBe( + legacy, + ); + await expect( + adoptEmbeddingArtifact( + f.root, + space, + next, + { ...rebased, fallbackLegacy: undefined }, + validate, + ), + ).rejects.toThrow('changed'); + const sealed = await adoptEmbeddingArtifact( + f.root, + space, + next, + { ...rebased, phase: 'sealed', fallbackLegacy: undefined }, + validate, + ); + expect(sealed.owner.fallbackLegacy).toBeUndefined(); + }); + it('streams a large exclusion list and is idempotent without copying the native source', async () => { + const f = await fixture(); + const before = await fs.stat(f.source); + let yielded = 0; + const ids = async function* () { + for (let i = 0; i < 12000; i++) { + yielded++; + yield `Const:a.ts:n${i}`; + } + }; + const identity = await recordLegacyEmbeddingSource(f.root, f.input, ids()); + expect(yielded).toBe(12000); + const retained = await readLegacyEmbeddingSource(f.root, identity); + expect(retained.recordedIdentity).toEqual(f.input.recordedIdentity); + expect(retained.recordedInputIdentity).toBeUndefined(); + expect((await collect(retained.excludedIds())).length).toBe(12000); + expect(await recordLegacyEmbeddingSource(f.root, f.input, ids())).toBe(identity); + expect((await fs.stat(f.source)).ino).toBe(before.ino); + expect((await fs.stat(f.source)).mtimeMs).toBe(before.mtimeMs); + expect( + (await fs.stat(path.join(f.root, 'embeddings', 'legacy', identity, 'legacy.json'))).size, + ).toBeLessThan(2048); + }); + it('preserves unknown fields and refuses paths outside a retired graph', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource( + f.root, + { ...f.input, recordedIdentity: { dimensions: 4 } }, + [], + ); + expect((await readLegacyEmbeddingSource(f.root, identity)).recordedIdentity).toEqual({ + dimensions: 4, + }); + await expect( + recordLegacyEmbeddingSource(f.root, { ...f.input, sourceFile: '../lbug' }, []), + ).rejects.toThrow('retired'); + await expect( + recordLegacyEmbeddingSource(f.root, { ...f.input, sourceFile: 'lbug' }, []), + ).rejects.toThrow('retired'); + }); + it('rejects replaced sources, source symlinks, and unrecovered WALs', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + await fs.rename(f.source, f.source + '.old'); + await fs.writeFile(f.source, 'retired database'); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow( + 'changed after retirement', + ); + await fs.unlink(f.source); + await fs.symlink(f.source + '.old', f.source); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('regular file'); + await fs.unlink(f.source); + await fs.rename(f.source + '.old', f.source); + const restoredIdentity = await recordLegacyEmbeddingSource(f.root, f.input, []); + await fs.writeFile(f.source + '.wal', 'not checkpointed'); + await expect(readLegacyEmbeddingSource(f.root, restoredIdentity)).rejects.toThrow( + 'native recovery', + ); + }); + it('rejects in-place source changes even when size and mtime are restored', async () => { + const f = await fixture(); + const timestamp = new Date('2020-01-01T00:00:00Z'); + await fs.utimes(f.source, timestamp, timestamp); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + const before = await fs.stat(f.source, { bigint: true }); + await fs.writeFile(f.source, 'changed database'); + await fs.utimes(f.source, timestamp, timestamp); + const after = await fs.stat(f.source, { bigint: true }); + expect({ ino: after.ino, size: after.size, mtime: after.mtimeNs }).toEqual({ + ino: before.ino, + size: before.size, + mtime: before.mtimeNs, + }); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow( + 'changed after retirement', + ); + expect(await fs.readFile(f.source, 'utf8')).toBe('changed database'); + }); + it('validates the complete exclusion checksum before a native index may become complete', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x']); + const file = path.join(f.root, 'embeddings', 'legacy', identity, 'excluded.jsonl'); + await fs.writeFile(file, '"Const:a.ts:y"\n'); + const retained = await readLegacyEmbeddingSource(f.root, identity); + await expect(collect(retained.excludedIds())).rejects.toThrow('checksum mismatch'); + await expect(recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x'])).rejects.toThrow( + 'checksum mismatch', + ); + }); + it('rejects a corrupt bounded receipt instead of selecting another source', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + await fs.writeFile(path.join(f.root, 'embeddings', 'legacy', identity, 'legacy.json'), '{}'); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('Malformed'); + }); + it('cleans only its unpublished spool on cancellation and retains source data', async () => { + const f = await fixture(); + const signal = AbortSignal.abort(); + await expect( + recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:x'], signal), + ).rejects.toThrow(); + expect(await fs.readFile(f.source, 'utf8')).toBe('retired database'); + expect(await fs.readdir(path.join(f.root, 'embeddings', 'legacy'))).toEqual([]); + }); + it('bounds corrupt exclusion framing and rejects directory symlinks', async () => { + const f = await fixture(); + const identity = await recordLegacyEmbeddingSource(f.root, f.input, []); + const directory = path.join(f.root, 'embeddings', 'legacy', identity); + await fs.rename(directory, directory + '.original'); + await fs.symlink(directory + '.original', directory); + await expect(readLegacyEmbeddingSource(f.root, identity)).rejects.toThrow('owned directory'); + await expect( + recordLegacyEmbeddingSource(f.root, f.input, ['Const:a.ts:' + 'x'.repeat(1024 * 1024)]), + ).rejects.toThrow('bounded size'); + }); +}); diff --git a/gitnexus/test/unit/local-backend-preserved-vectors.test.ts b/gitnexus/test/unit/local-backend-preserved-vectors.test.ts new file mode 100644 index 000000000..4fed67426 --- /dev/null +++ b/gitnexus/test/unit/local-backend-preserved-vectors.test.ts @@ -0,0 +1,106 @@ +import path from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const mocks = vi.hoisted(() => ({ metadata: vi.fn(), search: vi.fn(), hydrate: vi.fn() })); +vi.mock('../../src/storage/repo-manager.js', async (original) => ({ + ...(await original()), + loadMeta: mocks.metadata, +})); +vi.mock('../../src/core/lbug/pool-adapter.js', async (original) => ({ + ...(await original()), + executeParameterized: mocks.hydrate, +})); +vi.mock('../../src/core/embeddings/vector-generation-search.js', () => ({ + searchVectorGeneration: mocks.search, +})); +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const current = 'a'.repeat(32); +const source = 'b'.repeat(32); +const graph = path.join('/repo/.gitnexus', `lbug.generation.${current}`); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'new', + indexedAt: '', + graphPath: path.basename(graph), + graphGeneration: { version: 1, id: current, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity: 'd'.repeat(64), + artifactIdentity: 'e'.repeat(64), + graphGeneration: source, + preservedForGraph: current, + vectorIdentity: 'f'.repeat(64), + inputIdentity: '0'.repeat(64), + dimensions: 4, + nodes: 3, + chunks: 3, + indexKind: 'exact', + }, +}; +interface Reader { + semanticSearch(repo: { lbugPath: string }, query: string, limit: number): Promise; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.metadata.mockResolvedValue(metadata); + mocks.search.mockResolvedValue( + ['deleted', 'kept', 'second'].map((name, i) => ({ + nodeId: `Function:a.ts:${name}`, + chunkIndex: 0, + startLine: 1, + endLine: 2, + distance: i / 10, + })), + ); + mocks.hydrate.mockImplementation(async (file, _query, { nodeId }) => { + expect(file).toBe(graph); + return nodeId.endsWith(':deleted') + ? [] + : [{ name: nodeId.split(':').at(-1), filePath: 'current/a.ts' }]; + }); +}); + +describe('preserved semantic hits', () => { + it('hydrates only the current graph, drops deleted IDs and keeps surviving rank order', async () => { + const backend = new LocalBackend() as unknown as Reader; + const hits = await backend.semanticSearch({ lbugPath: graph }, 'example', 3); + expect(hits).toEqual([ + { + nodeId: 'Function:a.ts:kept', + name: 'kept', + type: 'Function', + filePath: 'current/a.ts', + distance: 0.1, + startLine: 1, + endLine: 2, + }, + { + nodeId: 'Function:a.ts:second', + name: 'second', + type: 'Function', + filePath: 'current/a.ts', + distance: 0.2, + startLine: 1, + endLine: 2, + }, + ]); + expect(mocks.search.mock.calls[0][0]).toEqual({ + metadataDirectory: path.dirname(graph), + metadata, + graphPath: graph, + }); + expect(mocks.hydrate).toHaveBeenCalledTimes(3); + }); + it('does not open the old vector artifact when a new job is pending', async () => { + mocks.metadata.mockResolvedValue({ + ...metadata, + embeddingJob: { version: 1, identity: '1'.repeat(64), graphGeneration: current }, + }); + const backend = new LocalBackend() as unknown as Reader; + expect(await backend.semanticSearch({ lbugPath: graph }, 'example', 3)).toEqual([]); + expect(mocks.search).not.toHaveBeenCalled(); + expect(mocks.hydrate).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/local-backend-vector-generation.test.ts b/gitnexus/test/unit/local-backend-vector-generation.test.ts new file mode 100644 index 000000000..135cbfc29 --- /dev/null +++ b/gitnexus/test/unit/local-backend-vector-generation.test.ts @@ -0,0 +1,151 @@ +import path from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const mocks = vi.hoisted(() => ({ query: vi.fn(), embed: vi.fn(), metadata: vi.fn() })); +vi.mock('../../src/core/lbug/pool-adapter.js', async (original) => ({ + ...(await original()), + executeQuery: mocks.query, +})); +vi.mock('../../src/storage/repo-manager.js', async (original) => ({ + ...(await original()), + loadMeta: mocks.metadata, +})); +vi.mock('../../src/mcp/core/embedder.js', () => ({ + embedQuery: mocks.embed, + getEmbeddingDims: () => 4, +})); +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const generation = 'a'.repeat(32); +const graphPath = path.join('/repo/.gitnexus', `lbug.generation.${generation}`); +const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: path.basename(graphPath), + graphGeneration: { version: 1, id: generation, coveredContent: 'b'.repeat(64) }, + embeddingJob: { version: 1, identity: 'c'.repeat(64), graphGeneration: generation }, +}; +interface Reader { + semanticSearch( + repo: { lbugPath: string; generationUnknown?: boolean }, + query: string, + limit: number, + degraded?: { reason?: string }, + ): Promise; + lastObservedPoolState: Map; + lastQueryEmbeddingDims: Map; + readToolMetadata(repo: { generationUnknown?: boolean }): Promise; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.query.mockResolvedValue([]); + mocks.metadata.mockResolvedValue(metadata); +}); + +describe('MCP generation gate', () => { + it.each([ + 'graph-generation-unverified', + 'vector-generation-mismatch', + 'vector-index-pending', + 'embeddings-empty', + 'embeddings-unavailable', + ])('reports semantic unavailability for %s except a never-embedded index', async (reason) => { + const backend = new LocalBackend() as unknown as Reader; + const meta: RepoMeta = { + ...metadata, + embeddingJob: undefined, + vectorGeneration: { + version: 1, + identity: 'a'.repeat(64), + graphGeneration: generation, + vectorIdentity: 'b'.repeat(64), + inputIdentity: 'c'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', + }, + }; + if (reason === 'graph-generation-unverified') meta.graphGeneration = undefined; + if (reason === 'vector-generation-mismatch') + meta.vectorGeneration!.graphGeneration = 'd'.repeat(32); + if (reason === 'vector-index-pending') meta.vectorGeneration!.chunks = 1_000_000_000; + if (reason === 'embeddings-empty') + Object.assign(meta.vectorGeneration!, { nodes: 0, chunks: 0 }); + if (reason === 'embeddings-unavailable') meta.vectorGeneration = undefined; + vi.spyOn(backend, 'readToolMetadata').mockResolvedValue(meta); + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch({ lbugPath: graphPath }, 'hello', 5, degraded)).toEqual([]); + if (reason === 'embeddings-unavailable') expect(degraded.reason).toBeUndefined(); + else { + expect(degraded.reason).toContain(reason); + expect(degraded.reason).toContain('gitnexus analyze'); + } + expect(mocks.embed).not.toHaveBeenCalled(); + }); + it('explains the temporary keyword-only lane while vectors are pending', async () => { + const backend = new LocalBackend() as unknown as Reader; + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch({ lbugPath: graphPath }, 'hello', 5, degraded)).toEqual([]); + expect(degraded.reason).toContain('embeddings are pending'); + expect(mocks.embed).not.toHaveBeenCalled(); + }); + it.each(['before-read', 'during-read'])( + 'clears a prior query width when ownership is lost %s', + async (when) => { + const backend = new LocalBackend() as unknown as Reader; + backend.lastQueryEmbeddingDims.set(graphPath, 4); + const repo = { lbugPath: graphPath, generationUnknown: when === 'before-read' }; + if (when === 'during-read') { + vi.spyOn(backend, 'readToolMetadata').mockImplementation(async (handle) => { + handle.generationUnknown = true; + return metadata; + }); + } + const degraded: { reason?: string } = {}; + expect(await backend.semanticSearch(repo, 'hello', 5, degraded)).toEqual([]); + expect(degraded.reason).toContain('graph generation is unverified'); + expect(degraded.reason).toContain('gitnexus analyze'); + expect(backend.lastQueryEmbeddingDims.has(graphPath)).toBe(false); + expect(mocks.embed).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }, + ); + + it.each(['pending', 'foreign-metadata', 'unknown-handle'])( + 'skips model and table access for %s', + async (state) => { + const backend = new LocalBackend() as unknown as Reader; + if (state === 'foreign-metadata') + mocks.metadata.mockResolvedValue({ + ...metadata, + graphPath: `lbug.generation.${'d'.repeat(32)}`, + }); + expect( + await backend.semanticSearch( + { lbugPath: graphPath, generationUnknown: state === 'unknown-handle' }, + 'hello', + 5, + ), + ).toEqual([]); + expect(mocks.query).not.toHaveBeenCalled(); + expect(mocks.embed).not.toHaveBeenCalled(); + }, + ); + it('keeps a warm reader on its own metadata when publication advances', async () => { + const backend = new LocalBackend() as unknown as Reader; + backend.lastObservedPoolState.set(graphPath, { metadata }); + mocks.metadata.mockResolvedValue({ + ...metadata, + graphPath: `lbug.generation.${'d'.repeat(32)}`, + graphGeneration: { ...metadata.graphGeneration, id: 'd'.repeat(32) }, + }); + const repo = { lbugPath: graphPath, generationUnknown: false }; + expect(await backend.semanticSearch(repo, 'hello', 5)).toEqual([]); + expect(repo.generationUnknown).toBe(false); + expect(mocks.query).not.toHaveBeenCalled(); + expect(mocks.embed).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/ops-snapshot.test.ts b/gitnexus/test/unit/ops-snapshot.test.ts index 298b823e1..6c55bad90 100644 --- a/gitnexus/test/unit/ops-snapshot.test.ts +++ b/gitnexus/test/unit/ops-snapshot.test.ts @@ -5,8 +5,38 @@ import { isGitNexusVercelOrigin, serializeOpsJob, summarizeOpsLane, + publicOpsProgress, } from '../../src/server/ops-snapshot.js'; import { publicRepoId } from '../../src/server/public-repo-id.js'; +import type { AnalyzeJobProgress } from '../../src/server/analyze-job.js'; + +it('copies only enumerated phases and finite numeric progress fields', () => { + const input = { + phase: 'embedding', + percent: 90, + message: 'Embedding', + graphPhase: '/private/path', + modelDownloadPercent: Infinity, + embeddingProgress: { + phase: 'embedding', + totalUnits: 'secret', + completedUnits: -1, + reusedUnits: NaN, + failedUnits: 1.5, + pendingUnits: 2, + activeUnit: '/secret', + chunks: 7, + identity: 'hidden', + endpoint: 'https://secret', + }, + } as unknown as AnalyzeJobProgress; + expect(publicOpsProgress(input)).toEqual({ + phase: 'embedding', + percent: 90, + message: 'Embedding', + embeddingProgress: { phase: 'embedding', pendingUnits: 2, chunks: 7 }, + }); +}); describe('serializeOpsJob / summarizeOpsLane', () => { let manager: JobManager; diff --git a/gitnexus/test/unit/parse-failure-drain.test.ts b/gitnexus/test/unit/parse-failure-drain.test.ts new file mode 100644 index 000000000..f5c368294 --- /dev/null +++ b/gitnexus/test/unit/parse-failure-drain.test.ts @@ -0,0 +1,47 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, it, vi } from 'vitest'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; + +const mocks = vi.hoisted(() => ({ + dispatch: vi.fn(async () => { + throw undefined; + }), + terminate: vi.fn(async () => { + throw new Error('drain failed'); + }), +})); +vi.mock('../../src/core/ingestion/workers/worker-pool.js', async (original) => ({ + ...(await original()), + createWorkerPool: () => ({ + size: 1, + dispatch: mocks.dispatch, + dispatchGroups: mocks.dispatch, + terminate: mocks.terminate, + }), +})); + +it('preserves an undefined parse rejection alongside a native drain failure', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-parse-drain-')); + const source = 'export function example() { return 1; }'; + try { + await fs.writeFile(path.join(root, 'a.ts'), source); + const run = runChunkedParseAndResolve( + createKnowledgeGraph(), + [{ path: 'a.ts', size: source.length }], + ['a.ts'], + 1, + root, + Date.now(), + () => {}, + { workerPoolSize: 1 }, + ); + await expect(run).rejects.toMatchObject({ errors: [undefined, new Error('drain failed')] }); + await expect(run).rejects.toThrow('undefined'); + expect(mocks.terminate).toHaveBeenCalledOnce(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +}); diff --git a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts index d9bd574bb..fa5d661af 100644 --- a/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts +++ b/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts @@ -341,6 +341,88 @@ describe('parse-impl worker pool lazy startup', () => { } }); + it('accepts a zero-worker budget when there are no parseable files', async () => { + const result = await runChunkedParseAndResolve( + createKnowledgeGraph(), + [], + [], + 0, + repoDir, + Date.now(), + () => {}, + { + workerPoolSize: 0, + workerResources: { workers: 0, workerHeapMb: 512 }, + workerUrlForTest: pathToFileURL(path.join(tempDir, 'must-not-spawn.cjs')), + }, + ); + expect(result.usedWorkerPool).toBe(false); + }); + + it('keeps tiny-repository automatic sizing inside the admitted worker ceiling', async () => { + const saved = process.env.GITNEXUS_WORKER_POOL_SIZE; + delete process.env.GITNEXUS_WORKER_POOL_SIZE; + try { + const rels = ['a.ts', 'b.ts', 'c.ts', 'd.ts']; + const scanned = rels.map((rel) => { + fs.writeFileSync(path.join(repoDir, rel), 'export const value = 1;'); + return { path: rel, size: 23 }; + }); + const markerDir = path.join(tempDir, 'budget-markers'); + const workerPath = path.join(tempDir, 'budget-worker.cjs'); + // Real worker reports V8's assigned old-generation limit. The generous + // default is several GiB on this host; this run must stay at 512 MiB. + fs.writeFileSync( + workerPath, + ` +const fs = require('node:fs'); +const { parentPort, resourceLimits, threadId } = require('node:worker_threads'); +fs.mkdirSync(${JSON.stringify(markerDir)}, { recursive: true }); +fs.writeFileSync(${JSON.stringify(markerDir)} + '/' + threadId, String(resourceLimits.maxOldGenerationSizeMb)); +parentPort.postMessage({ type: 'ready' }); +parentPort.on('message', (msg) => { + if (msg.type === 'sub-batch') parentPort.postMessage({ type: 'sub-batch-done' }); + if (msg.type === 'flush') parentPort.postMessage({ type: 'result', data: ${JSON.stringify(emptyWorkerResult('a.ts', 'value'))} }); +}); +`, + ); + await runChunkedParseAndResolve( + createKnowledgeGraph(), + scanned, + rels, + 4, + repoDir, + Date.now(), + () => {}, + { + workerUrlForTest: pathToFileURL(workerPath), + workerResources: { workers: 3, workerHeapMb: 512 }, + }, + ); + const markers = fs.readdirSync(markerDir); + expect(markers).toHaveLength(1); + expect(Number(fs.readFileSync(path.join(markerDir, markers[0]), 'utf8'))).toBe(512); + await expect( + runChunkedParseAndResolve( + createKnowledgeGraph(), + scanned, + rels, + 4, + repoDir, + Date.now(), + () => {}, + { + workerPoolSize: 4, + workerResources: { workers: 3, workerHeapMb: 512 }, + }, + ), + ).rejects.toThrow('admitted resource budget'); + } finally { + if (saved === undefined) delete process.env.GITNEXUS_WORKER_POOL_SIZE; + else process.env.GITNEXUS_WORKER_POOL_SIZE = saved; + } + }); + it('throws when GITNEXUS_WORKER_POOL_SIZE=0 and no --workers flag (sequential parsing removed)', async () => { const saved = process.env.GITNEXUS_WORKER_POOL_SIZE; process.env.GITNEXUS_WORKER_POOL_SIZE = '0'; diff --git a/gitnexus/test/unit/pipeline-runner.test.ts b/gitnexus/test/unit/pipeline-runner.test.ts index c29fdd7f7..4cd953bd4 100644 --- a/gitnexus/test/unit/pipeline-runner.test.ts +++ b/gitnexus/test/unit/pipeline-runner.test.ts @@ -18,6 +18,31 @@ function makeCtx(): PipelineContext { } describe('runPipeline', () => { + it('honors cancellation between native phases without entering the next phase', async () => { + const controller = new AbortController(); + const ctx: PipelineContext = { ...makeCtx(), options: { signal: controller.signal } }; + const entered: string[] = []; + const phases: PipelinePhase[] = [ + { + name: 'parse', + deps: [], + async execute() { + entered.push('parse'); + controller.abort(new Error('cancelled after parse')); + }, + }, + { + name: 'emit', + deps: ['parse'], + async execute() { + entered.push('emit'); + }, + }, + ]; + await expect(runPipeline(phases, ctx)).rejects.toThrow('cancelled after parse'); + expect(entered).toEqual(['parse']); + }); + it('executes phases in dependency order', async () => { const order: string[] = []; diff --git a/gitnexus/test/unit/rate-limit.test.ts b/gitnexus/test/unit/rate-limit.test.ts index 90e428b25..545efdca7 100644 --- a/gitnexus/test/unit/rate-limit.test.ts +++ b/gitnexus/test/unit/rate-limit.test.ts @@ -313,10 +313,10 @@ describe('production routes — rate-limit middleware wiring', () => { ); }); - it('embed route flushes WAL via flushWAL, not inline executeQuery (#1376)', () => { - // The embed handler must call the consolidated helper, not hand-roll - // its own try/catch around executeQuery('CHECKPOINT'). - expect(apiSource).toMatch(/await flushWAL\(\)/); + it('embed route delegates durable publication to the shared phase (#1376)', () => { + // The phase owns the vector writer and its durable checkpoint. The HTTP + // reader must not checkpoint its read-only graph connection. + expect(apiSource).toMatch(/await runAnalysisEmbeddingPhase\(/); expect(apiSource).not.toMatch(/executeQuery\('CHECKPOINT'\)/); }); }); diff --git a/gitnexus/test/unit/repo-projection.test.ts b/gitnexus/test/unit/repo-projection.test.ts index e58f27d8d..6d9a2a44c 100644 --- a/gitnexus/test/unit/repo-projection.test.ts +++ b/gitnexus/test/unit/repo-projection.test.ts @@ -171,6 +171,23 @@ describe('projectRepoListEntry — GET /api/repos', () => { }); describe('projectRepoDetail — GET /api/repo', () => { + it('reports graph completeness separately from pending vector work', () => { + const metadata = meta({ + scopeExtractionReceipt: 1, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + const out = projectRepoDetail(entry(), metadata, FRESH, FULL_SOURCE, { + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 1, + failedUnits: 1, + }); + expect(out.graphIncompleteReasons).toEqual([]); + expect(out.incompleteReasons).toEqual(['embedding-job-pending']); + expect(out.embeddingProgress).toMatchObject({ failedUnits: 1, pendingUnits: 1 }); + }); it('returns lastCommit and branch, which the route used to drop', () => { const out = projectRepoDetail(entry({ branch: 'master' }), null, FRESH, FULL_SOURCE); expect(out.lastCommit).toBe(entry().lastCommit); diff --git a/gitnexus/test/unit/resolve-invocation.test.ts b/gitnexus/test/unit/resolve-invocation.test.ts index 5d17736a6..bf835a525 100644 --- a/gitnexus/test/unit/resolve-invocation.test.ts +++ b/gitnexus/test/unit/resolve-invocation.test.ts @@ -376,6 +376,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { describe('warnIfNpm11NpxRisk (#1939 npm-11 nudge)', () => { afterEach(() => { vi.clearAllMocks(); + vi.unstubAllEnvs(); delete process.env.GITNEXUS_INVOCATION; }); @@ -452,12 +453,30 @@ describe('warnIfNpm11NpxRisk (#1939 npm-11 nudge)', () => { write.mockRestore(); }); + it.each(['11.0.0\n', '10.9.0\n', 'unreadable\n'])( + 'shares one npm probe with automatic runner selection for %s', + (version) => { + vi.stubEnv('GITNEXUS_INVOCATION', ''); + vi.stubEnv('PATH', ''); + mockedExec.mockReturnValue(version); + const write = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + try { + warnIfNpm11NpxRisk(); + expect(mockedExec).toHaveBeenCalledOnce(); + expect(write).toHaveBeenCalledTimes(version.startsWith('11.') ? 1 : 0); + } finally { + write.mockRestore(); + } + }, + ); + it('does not warn when a global gitnexus or pnpm is preferred', () => { process.env.GITNEXUS_INVOCATION = 'pnpm'; mockedExec.mockReturnValue('11.0.0\n'); const write = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); warnIfNpm11NpxRisk(); expect(write).not.toHaveBeenCalled(); + expect(mockedExec).not.toHaveBeenCalled(); write.mockRestore(); }); diff --git a/gitnexus/test/unit/resources.test.ts b/gitnexus/test/unit/resources.test.ts index 4d50a610d..c25619d0e 100644 --- a/gitnexus/test/unit/resources.test.ts +++ b/gitnexus/test/unit/resources.test.ts @@ -566,6 +566,23 @@ describe('context resource freshness after out-of-process analyze (#2438)', () = ); }); + it('separates a complete graph from an unverified pending embedding journal', async () => { + loadMetaMock.mockResolvedValue({ + repoPath: '/tmp/test-repo', + lastCommit: 'current-head', + indexedAt: '', + scopeExtractionReceipt: 1, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + const result = await readResource( + 'gitnexus://repo/test-project/context', + createMockBackend({ context: CONTEXT }), + ); + expect(result).toContain('incomplete_reasons: ["embedding-job-pending"]'); + expect(result).toContain('graph_incomplete_reasons: []'); + expect(result).toContain('embedding_progress: {"phase":"unverified"}'); + }); + it('exposes Spring Actuator index metadata to agents', async () => { loadMetaMock.mockResolvedValue({ repoPath: '/tmp/test-repo', diff --git a/gitnexus/test/unit/server-sse-payload.test.ts b/gitnexus/test/unit/server-sse-payload.test.ts index 0b11766c2..94384026e 100644 --- a/gitnexus/test/unit/server-sse-payload.test.ts +++ b/gitnexus/test/unit/server-sse-payload.test.ts @@ -16,7 +16,7 @@ * MCP wiring, which is what made this file cost ~25s to import (#2790 review). */ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { JobManager } from '../../src/server/analyze-job.js'; +import type { JobManager, AnalyzeJobProgress } from '../../src/server/analyze-job.js'; import { publicRepoId } from '../../src/server/public-repo-id.js'; import { startSSEHarness, terminalFrame, type SSEHarness } from '../helpers/sse-harness.js'; @@ -37,6 +37,47 @@ describe('mountSSEProgress terminal payload', () => { afterEach(() => harness.close()); + it('keeps graph-ready nonterminal, forwards safe counts, and reports the eventual vector failure once', async () => { + const job = manager.createJob({ repoPath: REPO_PATH }); + manager.updateJob(job.id, { status: 'analyzing' }); + const response = await fetch(`${baseUrl}/api/analyze/${job.id}/progress`); + manager.updateJob(job.id, { + progress: { + phase: 'graph-ready', + percent: 90, + message: 'Graph available', + graphPhase: 'ready', + embeddingProgress: { + phase: 'embedding', + totalUnits: 3, + completedUnits: 1, + reusedUnits: 0, + pendingUnits: 2, + failedUnits: 0, + activeUnit: 1, + sourcePath: '/private/source', + identity: 'private-key', + }, + modelDownloadPercent: 100, + secret: 'hidden-token', + } as AnalyzeJobProgress, + }); + expect(manager.getJob(job.id)?.status).toBe('analyzing'); + manager.updateJob(job.id, { status: 'failed', error: 'Embedding endpoint unavailable' }); + const body = await response.text(); + expect(body).toContain('"graphPhase":"ready"'); + expect(body).toContain('"activeUnit":1'); + expect(body).toContain('"pendingUnits":2'); + expect(body).toContain('"modelDownloadPercent":100'); + expect(body).not.toMatch(/private|hidden-token|sourcePath|identity/); + expect(body.match(/event: failed/g)).toHaveLength(1); + expect(body).not.toContain('event: complete'); + expect(terminalFrame(body, 'failed')).toEqual({ + repoName: 'reels', + error: 'Embedding endpoint unavailable', + }); + }); + it('already-terminal replay includes repoName and omits repoPath', async () => { const job = manager.createJob({ repoPath: REPO_PATH }); manager.updateJob(job.id, { status: 'complete', repoName: REPO_NAME }); diff --git a/gitnexus/test/unit/shared-store-generations.test.ts b/gitnexus/test/unit/shared-store-generations.test.ts new file mode 100644 index 000000000..16316919c --- /dev/null +++ b/gitnexus/test/unit/shared-store-generations.test.ts @@ -0,0 +1,531 @@ +import { execFile, execFileSync } from 'node:child_process'; +import { promisify } from 'node:util'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + ensurePrivateSharedGraph, + featureKeyOf, + publishSharedGraph, + seedSharedSlot, +} from '../../src/core/shared-store-analyze.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { + describeSharedGraph, + reclaimSharedStore, +} from '../../src/storage/shared-store-lifecycle.js'; +import { + resolveGraphPath, + resolveGraphWritePath, + sharedStoreLayout, +} from '../../src/storage/shared-store.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; +import { acquireIndexLock } from '../../src/storage/index-lock.js'; +import { + acquireAnalysisSlotLock, + sharedMemberGatePath, +} from '../../src/storage/analysis-slot-lock.js'; + +it('reports named slot-local generations as private and commit graphs as shared', () => { + const slot = path.resolve('store/checkouts/owner'); + expect(describeSharedGraph(path.join(slot, 'lbug'), slot)).toBe('private'); + expect(describeSharedGraph(path.join(slot, `lbug.generation.${'a'.repeat(32)}`), slot)).toBe( + 'private', + ); + expect(describeSharedGraph(path.resolve('store/commits/commit-feature/lbug'), slot)).toBe( + 'shared', + ); + expect(describeSharedGraph(path.resolve('store/checkouts/owner-other/lbug'), slot)).toBe( + 'shared', + ); +}); + +describe('shared-store logical generations', () => { + let root: string; + let repo: string; + let head: string; + let slot: string; + let layout: ReturnType; + const generation = `lbug.generation.${'a'.repeat(32)}`; + const git = (...args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8', stdio: 'pipe' }).trim(); + const metadata = (extra: Partial = {}): RepoMeta => ({ + repoPath: repo, + storagePath: slot, + lastCommit: head, + indexedAt: '2026-11-04T00:00:00Z', + parseCacheIdentity: { base: 'fixture-parser' }, + ...extra, + }); + const modern = (extra: Partial = {}): RepoMeta => + metadata({ + graphGeneration: { version: 1, id: 'a'.repeat(32), coveredContent: 'c'.repeat(64) }, + ...extra, + }); + const requireMeta = async (directory: string): Promise => { + const meta = await loadMeta(directory); + if (!meta) throw new Error(`Missing fixture metadata: ${directory}`); + return meta; + }; + const vector = (): NonNullable => ({ + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', + }); + const vectorFile = (directory: string) => + path.join(directory, 'embeddings', 'b'.repeat(64), 'vectors'); + const writeVector = async (directory: string) => { + const file = vectorFile(directory); + await fs.mkdir(path.dirname(file), { recursive: true }); + // These fixtures test filesystem ownership; native seals are tested in + // embedding-vector-store.test.ts and the published-job controller suite. + await fs.writeFile(file, 'sealed vector filesystem fixture'); + return file; + }; + + beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-shared-generations-'))); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + repo = path.join(root, 'repo'); + await fs.mkdir(repo); + git('init', '-q', '-b', 'main'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '--allow-empty', '-qm', 'fixture'); + head = git('rev-parse', 'HEAD'); + layout = sharedStoreLayout('fixture-0123456789ab', repo); + slot = layout.checkoutSlot; + await fs.mkdir(slot, { recursive: true }); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('shares modern graph facts independently of logical IDs and separate vector work', () => { + const a = modern({ stats: { embeddings: 0 } }); + const b = modern({ + graphGeneration: { version: 1, coveredContent: 'c'.repeat(64), id: 'f'.repeat(32) }, + embeddingDims: 4, + vectorGeneration: vector(), + embeddingJob: { version: 1, identity: '1'.repeat(64), graphGeneration: 'f'.repeat(32) }, + legacyVectorSource: '2'.repeat(64), + stats: { embeddings: 1 }, + }); + expect(featureKeyOf(a)).toBe(featureKeyOf(b)); + expect( + featureKeyOf({ + ...b, + graphGeneration: { version: 1, id: 'f'.repeat(32), coveredContent: '9'.repeat(64) }, + }), + ).not.toBe(featureKeyOf(a)); + expect(featureKeyOf({ ...b, schemaFingerprint: 'different graph schema' })).not.toBe( + featureKeyOf(a), + ); + expect(featureKeyOf(metadata({ stats: { embeddings: 0 } }))).not.toBe( + featureKeyOf(metadata({ stats: { embeddings: 1 } })), + ); + }); + + it('publishes a completed vector pin without copying its contents', async () => { + await fs.writeFile(path.join(slot, generation), 'completed graph'); + const original = await writeVector(slot); + await saveMeta( + slot, + modern({ graphPath: generation, vectorGeneration: vector(), stats: { embeddings: 1 } }), + ); + const copy = vi.spyOn(fs, 'copyFile'); + await publishSharedGraph(layout, repo, head, () => {}); + const committed = path.dirname(resolveGraphPath(slot)); + expect((await loadMeta(committed))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(committed))).ino).toBe((await fs.stat(original)).ino); + expect(copy).not.toHaveBeenCalled(); + }); + + it('keeps the private graph while historical vectors still belong to its migration', async () => { + const graph = path.join(slot, generation); + await fs.writeFile(graph, 'graph with preserved legacy vectors'); + const meta = modern({ + graphPath: generation, + legacyVectorSource: 'e'.repeat(64), + embeddingDims: 4, + stats: { embeddings: 3 }, + }); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(graph); + expect(await loadMeta(slot)).toMatchObject(meta); + expect(await fs.readFile(graph, 'utf8')).toBe('graph with preserved legacy vectors'); + }); + + it('does not collect an orphan member while a branch writer owns its lock', async () => { + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + const lock = await acquireIndexLock(branch, { sweep: false }); + try { + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([]); + expect((await fs.stat(branch)).isDirectory()).toBe(true); + } finally { + lock.release(); + } + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([slot]); + }); + + it('admits new branch writers under the member gate, then releases it for sibling work', async () => { + const gate = await acquireIndexLock(slot, { sweep: false }); + const branch = path.join(slot, 'branches', 'new-branch'); + try { + await expect( + acquireAnalysisSlotLock(branch, { timeoutMs: 1, sweep: false }), + ).rejects.toThrow(); + } finally { + gate.release(); + } + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1000, sweep: false }); + try { + const rootWriter = await acquireIndexLock(slot, { timeoutMs: 1000, sweep: false }); + rootWriter.release(); + await expect(acquireIndexLock(branch, { timeoutMs: 1, sweep: false })).rejects.toThrow(); + } finally { + writer.release(); + } + }); + + it('uses the same file-backed admission gate through a storage-home alias', async () => { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + const alias = path.join(root, 'home-alias'); + await fs.symlink(path.join(root, 'home'), alias, 'junction'); + const gate = await acquireIndexLock(sharedMemberGatePath(layout.root, slot), { sweep: false }); + try { + vi.stubEnv('GITNEXUS_HOME', alias); + const branch = path.join( + alias, + 'stores', + path.basename(layout.root), + 'checkouts', + path.basename(slot), + 'branches', + 'aliased', + ); + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1, sweep: false }).catch( + () => null, + ); + writer?.release(); + expect(writer).toBeNull(); + } finally { + gate.release(); + } + }); + + it.each([ + { stage: 'member', kind: 'flat' }, + { stage: 'graph', kind: 'flat' }, + { stage: 'store', kind: 'flat' }, + { stage: 'member', kind: 'branch' }, + { stage: 'graph', kind: 'branch' }, + { stage: 'store', kind: 'branch' }, + ])('keeps $kind admission closed throughout orphan $stage deletion', async ({ stage, kind }) => { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + const commit = path.join(layout.commitsDir, 'unreferenced'); + await fs.mkdir(commit, { recursive: true }); + await fs.writeFile(path.join(commit, 'lbug'), 'old graph'); + const branch = kind === 'flat' ? slot : path.join(slot, 'branches', 'new-branch'); + const remove = fs.rm.bind(fs); + let checked = false; + let admission: string | undefined; + vi.spyOn(fs, 'rm').mockImplementation(async (target, options) => { + await remove(target, options); + const deleting = stage === 'member' ? slot : stage === 'graph' ? commit : layout.root; + if (String(target) !== deleting) return; + checked = true; + // A second process must consult the lock on disk, not this process's + // in-memory lock queue, after GC has removed the old member directory. + const moduleUrl = new URL('../../src/storage/analysis-slot-lock.ts', import.meta.url).href; + const { stdout } = await promisify(execFile)(process.execPath, [ + '--import', + 'tsx', + '--input-type=module', + '--eval', + `import { acquireAnalysisSlotLock } from ${JSON.stringify(moduleUrl)}; + const lock = await acquireAnalysisSlotLock(${JSON.stringify(branch)}, { timeoutMs: 1, sweep: false }).catch(() => null); + console.log(lock ? 'admitted' : 'blocked'); + lock?.release();`, + ]); + admission = stdout.trim(); + }); + const result = await reclaimSharedStore(layout.root, { gc: true }); + expect(checked).toBe(true); + expect(admission).toBe('blocked'); + expect(result.droppedMembers).toEqual([slot]); + expect(result.storeRemoved).toBe(true); + const writer = await acquireAnalysisSlotLock(branch, { timeoutMs: 1000, sweep: false }); + writer.release(); + }); + + it.each(['e'.repeat(64), null])( + 'retains an orphan with historical vector evidence: %s', + async (source) => { + await saveMeta( + slot, + metadata({ + repoPath: path.join(root, 'removed-checkout'), + legacyVectorSource: source as unknown as string, + }), + ); + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([]); + expect(await loadMeta(slot)).toHaveProperty('legacyVectorSource', source); + }, + ); + + it('seeds independent completed vector pins that survive collection of their source', async () => { + await fs.writeFile(path.join(slot, generation), 'completed graph'); + const original = await writeVector(slot); + const inode = (await fs.stat(original)).ino; + await saveMeta( + slot, + modern({ graphPath: generation, vectorGeneration: vector(), stats: { embeddings: 1 } }), + ); + await publishSharedGraph(layout, repo, head, () => {}); + const sibling = path.join(root, 'sibling'); + git('worktree', 'add', '-q', '-b', 'sibling', sibling); + const next = sharedStoreLayout(path.basename(layout.root), sibling); + await seedSharedSlot(next, sibling, () => {}); + expect((await fs.stat(vectorFile(next.checkoutSlot))).ino).toBe(inode); + const seeded = await requireMeta(next.checkoutSlot); + expect(seeded?.vectorGeneration).toEqual(vector()); + await fs.writeFile(path.join(next.checkoutSlot, generation), 'equivalent private graph'); + await saveMeta(next.checkoutSlot, { ...seeded, graphPath: generation }); + await fs.rm(slot, { recursive: true, force: true }); + expect((await reclaimSharedStore(layout.root)).removed).toHaveLength(1); + expect(await fs.readFile(vectorFile(next.checkoutSlot), 'utf8')).toBe( + 'sealed vector filesystem fixture', + ); + }); + + it('keeps pending job ownership in its checkout when publishing graph facts', async () => { + const job = { version: 1 as const, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }; + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await saveMeta(slot, modern({ graphPath: generation, embeddingJob: job })); + await publishSharedGraph(layout, repo, head, () => {}); + expect((await loadMeta(slot))?.embeddingJob).toEqual(job); + const committed = await loadMeta(path.dirname(resolveGraphPath(slot))); + expect(committed?.graphGeneration).toEqual(modern().graphGeneration); + expect(committed?.embeddingJob).toBeUndefined(); + expect(committed?.legacyVectorSource).toBeUndefined(); + }); + + it('adds completed vectors to an already published graph without replacing that graph', async () => { + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await saveMeta(slot, modern({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + const graph = resolveGraphPath(slot); + const graphInode = (await fs.stat(graph)).ino; + const original = await writeVector(slot); + const before = await requireMeta(slot); + await saveMeta(slot, { ...before, vectorGeneration: vector(), stats: { embeddings: 1 } }); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(graph); + expect((await fs.stat(graph)).ino).toBe(graphInode); + expect((await loadMeta(path.dirname(graph)))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(path.dirname(graph)))).ino).toBe( + (await fs.stat(original)).ino, + ); + }); + + it('adopts a completed local generation with its vectors while retaining the source', async () => { + const source = path.join(repo, '.gitnexus'); + await fs.mkdir(source); + await fs.writeFile(path.join(source, generation), 'local generation'); + const original = await writeVector(source); + await saveMeta( + source, + modern({ + storagePath: source, + graphPath: generation, + vectorGeneration: vector(), + stats: { embeddings: 1 }, + }), + ); + await seedSharedSlot(layout, repo, () => {}); + expect(await fs.readFile(resolveGraphPath(slot), 'utf8')).toBe('local generation'); + expect((await loadMeta(slot))?.vectorGeneration).toEqual(vector()); + expect((await fs.stat(vectorFile(slot))).ino).toBe((await fs.stat(original)).ino); + expect(await fs.readFile(path.join(source, generation), 'utf8')).toBe('local generation'); + }); + + it('retains historical coverage and the exact-scan limit in shared metadata', async () => { + await fs.writeFile(path.join(slot, generation), 'ready graph'); + await writeVector(slot); + const completed = { + ...vector(), + graphGeneration: 'f'.repeat(32), + preservedForGraph: 'a'.repeat(32), + chunks: 5, + }; + const meta = modern({ + graphPath: generation, + vectorGeneration: completed, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb', status: 'available' }, + vectorSearch: { provider: 'exact-scan', status: 'exact-scan', exactScanLimit: 2 }, + }, + }); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + const published = await requireMeta(path.dirname(resolveGraphPath(slot))); + expect(published.vectorGeneration).toEqual(completed); + expect(published.capabilities?.vectorSearch).toMatchObject({ + status: 'unavailable', + reason: 'vector-index-pending', + coverage: 'preserved', + sourceGraphGeneration: 'f'.repeat(32), + exactScanLimit: 2, + }); + expect(featureKeyOf(published)).toBe(featureKeyOf(meta)); + }); + + it('keeps a modern private graph when an existing shared graph has no metadata', async () => { + const meta = modern({ graphPath: generation }); + const target = path.join(layout.commitsDir, `${head}-${featureKeyOf(meta)}`); + await fs.mkdir(target, { recursive: true }); + await fs.writeFile(path.join(target, 'lbug'), 'unverified shared graph'); + await fs.writeFile(path.join(slot, generation), 'verified private graph'); + await saveMeta(slot, meta); + await publishSharedGraph(layout, repo, head, () => {}); + expect(resolveGraphPath(slot)).toBe(path.join(slot, generation)); + expect(await fs.readFile(resolveGraphPath(slot), 'utf8')).toBe('verified private graph'); + }); + + it("refuses to strand this checkout's pending local job during adoption", async () => { + const source = path.join(repo, '.gitnexus'); + await fs.mkdir(source); + await fs.writeFile(path.join(source, generation), 'local generation'); + const job = { version: 1 as const, identity: 'b'.repeat(64), graphGeneration: 'a'.repeat(32) }; + await saveMeta( + source, + modern({ storagePath: source, graphPath: generation, embeddingJob: job }), + ); + await expect(seedSharedSlot(layout, repo, () => {})).rejects.toThrow('pending embedding job'); + expect((await loadMeta(source))?.embeddingJob).toEqual(job); + expect(await loadMeta(slot)).toBeNull(); + }); + + it.each([true, false])( + 'keeps an owned generation in place without copying or dropping its pointer (copy=%s)', + async (copy) => { + const file = path.join(slot, generation); + await fs.writeFile(file, 'private graph'); + await saveMeta(slot, metadata({ graphPath: generation })); + const before = await fs.stat(file); + const copyFile = vi.spyOn(fs, 'copyFile'); + const log = vi.fn(); + expect(await ensurePrivateSharedGraph(slot, log, { copy })).toBe(true); + expect(resolveGraphPath(slot)).toBe(file); + expect((await fs.stat(file)).ino).toBe(before.ino); + await expect(fs.stat(path.join(slot, 'lbug'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(copyFile).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + }, + ); + + it('routes only immutable shared graphs to a new private write target', async () => { + const commitGraph = path.join(layout.commitsDir, `${head}-deadbeef`, 'lbug'); + await saveMeta(slot, metadata({ graphPath: commitGraph })); + expect(resolveGraphWritePath(slot)).toBe(path.join(slot, 'lbug')); + await fs.writeFile(path.join(slot, generation), 'private graph'); + await saveMeta(slot, metadata({ graphPath: generation })); + expect(resolveGraphWritePath(slot)).toBe(path.join(slot, generation)); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await fs.writeFile(path.join(branch, generation), 'branch graph'); + await saveMeta(branch, metadata({ graphPath: generation })); + expect(resolveGraphWritePath(branch)).toBe(path.join(branch, generation)); + }); + + it('publishes a completed private generation by moving its file, preserving the inode', async () => { + const file = path.join(slot, generation); + await fs.writeFile(file, 'completed graph'); + const before = await fs.stat(file); + await saveMeta(slot, metadata({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + const published = resolveGraphPath(slot); + expect(path.dirname(path.dirname(published))).toBe(layout.commitsDir); + expect((await fs.stat(published)).ino).toBe(before.ino); + expect(await fs.readFile(published, 'utf8')).toBe('completed graph'); + }); + + it('keeps an unshareable private generation pointer intact', async () => { + await fs.writeFile(path.join(slot, generation), 'dirty graph'); + await fs.writeFile(path.join(slot, 'lbug'), 'obsolete canonical graph'); + await fs.writeFile(path.join(repo, 'untracked.ts'), 'export const changed = true;'); + await saveMeta(slot, metadata({ graphPath: generation })); + await publishSharedGraph(layout, repo, head, () => {}); + expect((await loadMeta(slot))?.graphPath).toBe(generation); + expect(resolveGraphPath(slot)).toBe(path.join(slot, generation)); + }); + + it.each(['prepared', 'pending', 'branch-prepared', 'branch-pending'])( + 'retains an orphan slot with %s analysis work', + async (kind) => { + const missingRepo = path.join(root, 'removed-checkout'); + await saveMeta(slot, metadata({ repoPath: missingRepo })); + const work = kind.startsWith('branch-') ? path.join(slot, 'branches', 'feature') : slot; + await fs.mkdir(work, { recursive: true }); + if (kind.endsWith('prepared')) { + // Even an unreadable/incomplete publication receipt is retained evidence. + await fs.writeFile(path.join(work, 'analysis-publication.json'), '{}'); + } else { + await saveMeta( + work, + metadata({ + repoPath: missingRepo, + embeddingJob: { + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + }, + }), + ); + } + await fs.writeFile(path.join(work, 'paid-work'), 'retained'); + const result = await reclaimSharedStore(layout.root, { gc: true }); + expect(result.droppedMembers).toEqual([]); + expect(await fs.readFile(path.join(work, 'paid-work'), 'utf8')).toBe('retained'); + await fs.rm(path.join(work, 'analysis-publication.json'), { force: true }); + await saveMeta(work, metadata({ repoPath: missingRepo })); + expect((await reclaimSharedStore(layout.root, { gc: true })).droppedMembers).toEqual([slot]); + await expect(fs.stat(slot)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('counts retained branch references before collecting shared graphs', async () => { + const graph = path.join(layout.commitsDir, `${head}-deadbeef`, 'lbug'); + await fs.mkdir(path.dirname(graph), { recursive: true }); + await fs.writeFile(graph, 'retained graph'); + await saveMeta(slot, metadata()); + const branch = path.join(slot, 'branches', 'feature'); + await fs.mkdir(branch, { recursive: true }); + await saveMeta(branch, metadata({ graphPath: graph })); + expect((await reclaimSharedStore(layout.root)).removed).toEqual([]); + expect(await fs.readFile(graph, 'utf8')).toBe('retained graph'); + }); + + it('aborts collection when branch ownership cannot be inspected', async () => { + await saveMeta(slot, metadata({ repoPath: path.join(root, 'removed-checkout') })); + await fs.writeFile(path.join(slot, 'branches'), 'unverified branch state'); + await expect(reclaimSharedStore(layout.root, { gc: true })).rejects.toThrow( + 'unverified branches directory', + ); + expect(await fs.readFile(path.join(slot, 'branches'), 'utf8')).toBe('unverified branch state'); + }); +}); diff --git a/gitnexus/test/unit/shared-store-read-security.test.ts b/gitnexus/test/unit/shared-store-read-security.test.ts new file mode 100644 index 000000000..dd235a418 --- /dev/null +++ b/gitnexus/test/unit/shared-store-read-security.test.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { resolveGraphPath } from '../../src/storage/shared-store.js'; +import { INDEX_METADATA_FILE } from '../../src/storage/storage-constants.js'; + +const directories: string[] = []; +const first = `lbug.generation.${'a'.repeat(32)}`; +const second = `lbug.generation.${'b'.repeat(32)}`; + +function fixture() { + const slot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gn-graph-read-'))); + directories.push(slot); + const metadata = path.join(slot, INDEX_METADATA_FILE); + fs.writeFileSync(metadata, JSON.stringify({ graphPath: first })); + fs.writeFileSync(path.join(slot, first), 'first graph'); + fs.writeFileSync(path.join(slot, second), 'second graph'); + return { slot, metadata }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.doUnmock('fs'); + vi.resetModules(); + for (const directory of directories.splice(0)) + fs.rmSync(directory, { recursive: true, force: true }); +}); + +it('keeps warm metadata reads to one stat without opening or parsing the file', () => { + const { slot, metadata } = fixture(); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + const open = vi.spyOn(fs, 'openSync'); + const read = vi.spyOn(fs, 'readFileSync'); + const parse = vi.spyOn(JSON, 'parse'); + const stat = vi.spyOn(fs, 'lstatSync'); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + expect(open).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + expect(parse).not.toHaveBeenCalled(); + expect(stat.mock.calls.filter(([file]) => file === metadata)).toHaveLength(1); +}); + +it('reads the opened metadata snapshot and invalidates it after pathname replacement', () => { + const { slot, metadata } = fixture(); + const replacement = path.join(slot, 'replacement.json'); + fs.writeFileSync(replacement, JSON.stringify({ graphPath: second })); + // Model a pathname swap while keeping both real files available: Windows + // refuses rename-over-open, but descriptor reads must remain pinned there too. + let replaced = false; + const open = fs.openSync.bind(fs); + const stat = fs.lstatSync.bind(fs); + vi.spyOn(fs, 'openSync').mockImplementation((file, flags, mode) => + open(replaced && file === metadata ? replacement : file, flags, mode), + ); + vi.spyOn(fs, 'lstatSync').mockImplementation((file, options) => + stat(replaced && file === metadata ? replacement : file, options), + ); + const read = fs.readFileSync.bind(fs); + const spy = vi.spyOn(fs, 'readFileSync').mockImplementation((...args) => { + replaced = true; + if (args[0] === metadata) args[0] = replacement; + return read(...args); + }); + expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + expect(resolveGraphPath(slot)).toBe(path.join(slot, second)); + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.calls.every(([file]) => typeof file === 'number')).toBe(true); +}); + +it('preserves a relative storage path while validating its generation', () => { + const { slot } = fixture(); + const relative = path.relative(process.cwd(), slot); + expect(resolveGraphPath(relative, { graphPath: first })).toBe(path.join(relative, first)); +}); + +it.skipIf(process.platform === 'win32').each([false, true])( + 'rejects symlinked metadata with a warm cache of %s', + (warm) => { + const { slot, metadata } = fixture(); + const outside = fixture(); + if (warm) expect(resolveGraphPath(slot)).toBe(path.join(slot, first)); + fs.unlinkSync(metadata); + fs.symlinkSync(outside.metadata, metadata); + const read = vi.spyOn(fs, 'readFileSync'); + expect(resolveGraphPath(slot)).toBe(path.join(slot, 'lbug')); + expect(read).not.toHaveBeenCalled(); + }, +); + +it.skipIf(process.platform === 'win32')( + 'rejects a metadata symlink when O_NOFOLLOW is unavailable and closes the descriptor', + async () => { + const { slot, metadata } = fixture(); + const outside = fixture(); + vi.doMock('fs', async (original) => { + const actual = await original(); + return { + ...actual, + default: { ...fs, constants: { ...fs.constants, O_NOFOLLOW: undefined } }, + }; + }); + const open = fs.openSync.bind(fs); + vi.spyOn(fs, 'openSync').mockImplementation((file, flags, mode) => { + if (file === metadata) { + fs.unlinkSync(metadata); + fs.symlinkSync(outside.metadata, metadata); + } + return open(file, flags, mode); + }); + const close = vi.spyOn(fs, 'closeSync'); + const read = vi.spyOn(fs, 'readFileSync'); + vi.resetModules(); + const { resolveGraphPath: resolveWithoutNoFollow } = + await import('../../src/storage/shared-store.js'); + expect(resolveWithoutNoFollow(slot)).toBe(path.join(slot, 'lbug')); + expect(read).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + }, +); + +it.each(['../outside', '/outside/lbug', '..\\outside', `../${first}`])( + 'rejects an escaping graph pointer %s without accessing it', + (graphPath) => { + const { slot } = fixture(); + const stat = vi.spyOn(fs, 'lstatSync'); + expect(resolveGraphPath(slot, { graphPath })).toBe(path.join(slot, 'lbug')); + expect(stat).not.toHaveBeenCalled(); + }, +); diff --git a/gitnexus/test/unit/shared-vector-artifact.test.ts b/gitnexus/test/unit/shared-vector-artifact.test.ts new file mode 100644 index 000000000..df0399dd1 --- /dev/null +++ b/gitnexus/test/unit/shared-vector-artifact.test.ts @@ -0,0 +1,159 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { retainSharedVectorArtifact } from '../../src/storage/shared-vector-artifact.js'; +import type { PublishedVectorGeneration } from '../../src/storage/vector-generation.js'; + +const generation: PublishedVectorGeneration = { + version: 1, + identity: 'b'.repeat(64), + graphGeneration: 'a'.repeat(32), + vectorIdentity: 'c'.repeat(64), + inputIdentity: 'd'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 1, + indexKind: 'exact', +}; +let root: string, + source: string, + destination: string, + original: string, + retained: string, + preparing: string; +const differentDevice = () => Object.assign(new Error('Different filesystem'), { code: 'EXDEV' }); +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-retain-vectors-'))); + source = path.join(root, 'source'); + destination = path.join(root, 'destination'); + original = path.join(source, 'embeddings', generation.identity, 'vectors'); + retained = path.join(destination, 'embeddings', generation.identity, 'vectors'); + preparing = path.join(destination, 'embeddings', `.retain-${generation.identity}`); + await fs.mkdir(path.dirname(original), { recursive: true }); + await fs.mkdir(destination); + // Filesystem ownership fixture; native immutability/identity is covered by + // embedding-vector-store.test.ts and embedding-job-controller.test.ts. + await fs.writeFile(original, 'completed immutable vectors'); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +it('retains an independent hard link without copying or rewriting it on retry', async () => { + const copy = vi.spyOn(fs, 'copyFile'); + await retainSharedVectorArtifact(source, destination, generation); + const inode = (await fs.stat(retained)).ino; + expect(inode).toBe((await fs.stat(original)).ino); + await retainSharedVectorArtifact(source, destination, generation); + expect((await fs.stat(retained)).ino).toBe(inode); + expect(copy).not.toHaveBeenCalled(); + await fs.unlink(original); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('copies only for explicit initial adoption across filesystems, then reuses the receipt', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + await retainSharedVectorArtifact(source, destination, generation, true); + expect((await fs.stat(retained)).ino).not.toBe((await fs.stat(original)).ino); + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledOnce(); + expect(await fs.readFile(retained, 'utf8')).toBe(await fs.readFile(original, 'utf8')); +}); + +it('retains its source and refuses an unadmitted cross-device copy', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toMatchObject({ + code: 'EXDEV', + }); + expect(copy).not.toHaveBeenCalled(); + expect(await fs.readFile(original, 'utf8')).toBe('completed immutable vectors'); + await expect(fs.stat(retained)).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('retries a partial initial copy without overwriting its immutable source', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile').mockImplementationOnce(async (_from, to) => { + await fs.writeFile(to, 'partial'); + throw Object.assign(new Error('Disk full'), { code: 'ENOSPC' }); + }); + await expect( + retainSharedVectorArtifact(source, destination, generation, true), + ).rejects.toMatchObject({ code: 'ENOSPC' }); + expect(await fs.readFile(original, 'utf8')).toBe('completed immutable vectors'); + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledTimes(2); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('publishes an already durable copy after interruption without copying again', async () => { + vi.spyOn(fs, 'link').mockRejectedValue(differentDevice()); + const copy = vi.spyOn(fs, 'copyFile'); + const rename = fs.rename; + let interrupted = false; + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + if (from === preparing && !interrupted) { + interrupted = true; + throw new Error('Interrupted before publication'); + } + return rename(from, to); + }); + await expect(retainSharedVectorArtifact(source, destination, generation, true)).rejects.toThrow( + 'Interrupted', + ); + const inode = (await fs.stat(path.join(preparing, 'vectors'))).ino; + await retainSharedVectorArtifact(source, destination, generation, true); + expect(copy).toHaveBeenCalledOnce(); + expect((await fs.stat(retained)).ino).toBe(inode); +}); + +it('does not replace an existing artifact without a matching retention receipt', async () => { + await fs.mkdir(path.dirname(retained), { recursive: true }); + await fs.writeFile(retained, 'foreign or pending artifact'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow(); + expect(await fs.readFile(retained, 'utf8')).toBe('foreign or pending artifact'); +}); + +it('rejects an unfinished source without moving its sidecars', async () => { + await fs.writeFile(original + '.wal', 'pending native writes'); + const link = vi.spyOn(fs, 'link'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow( + 'unfinished native writes', + ); + expect(link).not.toHaveBeenCalled(); + expect(await fs.readFile(original + '.wal', 'utf8')).toBe('pending native writes'); +}); + +it('recovers a crash before the first intent record became durable', async () => { + await fs.mkdir(preparing, { recursive: true }); + await fs.writeFile( + path.join(preparing, `.retained.json.${randomUUID()}.tmp`), + 'incomplete record', + ); + await retainSharedVectorArtifact(source, destination, generation); + expect(await fs.readFile(retained, 'utf8')).toBe('completed immutable vectors'); +}); + +it('retains an unreadable preparation and its files as evidence', async () => { + await fs.mkdir(preparing, { recursive: true }); + await fs.writeFile(path.join(preparing, 'retained.json'), 'corrupt'); + await fs.writeFile(path.join(preparing, 'vectors'), 'unverified work'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow(); + expect(await fs.readFile(path.join(preparing, 'vectors'), 'utf8')).toBe('unverified work'); +}); + +it('retains an unacknowledged preparation with unexpected files', async () => { + await fs.mkdir(preparing, { recursive: true }); + const temporary = path.join(preparing, `.retained.json.${randomUUID()}.tmp`); + await fs.writeFile(temporary, 'incomplete record'); + await fs.writeFile(path.join(preparing, 'vectors'), 'unverified work'); + await expect(retainSharedVectorArtifact(source, destination, generation)).rejects.toThrow( + 'artifacts retained', + ); + expect(await fs.readFile(temporary, 'utf8')).toBe('incomplete record'); + expect(await fs.readFile(path.join(preparing, 'vectors'), 'utf8')).toBe('unverified work'); +}); diff --git a/gitnexus/test/unit/status-content-drift.test.ts b/gitnexus/test/unit/status-content-drift.test.ts index f6b131f20..65131dd14 100644 --- a/gitnexus/test/unit/status-content-drift.test.ts +++ b/gitnexus/test/unit/status-content-drift.test.ts @@ -115,6 +115,44 @@ afterEach(() => { }); describe('status freshness from per-file drift (#3077)', () => { + it('checks graph content despite pending embeddings and preserves the aggregate incomplete verdict', async () => { + vi.mocked(loadMeta).mockResolvedValue({ + ...repoWithCoverage.meta, + graphGeneration: { version: 1, id: 'b'.repeat(32), coveredContent: 'c'.repeat(64) }, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + vi.mocked(detectIndexContentDrift).mockResolvedValue({ kind: 'current', coveredFileCount: 1 }); + await statusCommand({ json: true }); + expect(detectIndexContentDrift).toHaveBeenCalled(); + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + index: { + graphStatus: 'up-to-date', + graphIncompleteReasons: [], + incompleteReasons: ['embedding-job-pending'], + embeddingProgress: { phase: 'unverified' }, + }, + }); + }); + + it('does not certify a drifted graph while embeddings are pending', async () => { + vi.mocked(loadMeta).mockResolvedValue({ + ...repoWithCoverage.meta, + embeddingJob: { version: 1, identity: 'a'.repeat(64), graphGeneration: 'b'.repeat(32) }, + }); + vi.mocked(detectIndexContentDrift).mockResolvedValue({ + kind: 'drifted', + changed: ['a.js'], + added: [], + deleted: [], + }); + await statusCommand({ json: true }); + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + index: { graphStatus: 'stale' }, + contentDrift: { status: 'drifted' }, + }); + }); it('is up-to-date when every covered file matches, despite a dirty working tree', async () => { // The reported case: one modified file outside the index's coverage. The // old repo-wide check called this stale and `analyze` could not clear it. diff --git a/gitnexus/test/unit/vector-generation-preservation.test.ts b/gitnexus/test/unit/vector-generation-preservation.test.ts new file mode 100644 index 000000000..7b904a47f --- /dev/null +++ b/gitnexus/test/unit/vector-generation-preservation.test.ts @@ -0,0 +1,282 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + beginInPlaceAnalysis, + prepareAnalysisPublication, + publishPreparedAnalysis, +} from '../../src/core/analysis-publication.js'; +import { loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { selectVectorGeneration } from '../../src/storage/vector-generation.js'; +import { readIndexCapabilities } from '../../src/storage/index-capabilities.js'; +import { readEmbeddingJobProgress } from '../../src/storage/embedding-job-progress.js'; +import { publicOpsProgress } from '../../src/server/ops-snapshot.js'; +import type { AnalyzeJobProgress } from '../../src/server/analyze-job.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const generation = 'a'.repeat(32); +const identity = 'b'.repeat(64); +const artifactIdentity = 'c'.repeat(64); +const roots: string[] = []; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'gn-preserved-'))); + roots.push(root); + const graphPath = `lbug.generation.${generation}`; + const metadata: RepoMeta = { + repoPath: '/repo', + lastCommit: 'old', + indexedAt: '', + runnerIdentity: { schemaVersion: 4 } as RepoMeta['runnerIdentity'], + graphPath, + graphGeneration: { version: 1, id: generation, coveredContent: 'd'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + artifactIdentity, + graphGeneration: generation, + vectorIdentity: 'e'.repeat(64), + inputIdentity: 'f'.repeat(64), + dimensions: 4, + nodes: 2, + chunks: 3, + indexKind: 'exact', + }, + }; + const graph = path.join(root, graphPath); + const vectors = path.join(root, 'embeddings', artifactIdentity, 'vectors'); + await fs.mkdir(path.dirname(vectors), { recursive: true }); + await fs.writeFile(vectors, 'untouched vector artifact'); + await fs.writeFile(graph, 'graph'); + await saveMeta(root, metadata); + const actions = { verify: async () => {}, saveMetadata: (m: RepoMeta) => saveMeta(root, m) }; + const stage = async () => { + const file = path.join(root, `lbug.staging.${randomUUID()}`); + await fs.writeFile(file, 'next graph'); + return file; + }; + return { root, metadata, graph, vectors, actions, stage }; +} +afterEach(async () => { + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +describe('preserved vector coverage', () => { + it.each(['../foreign/lbug', '/foreign/lbug', 'lbug.generation.invalid'])( + 'does not report vectors as ready for a rejected graph route: %s', + async (graphPath) => { + const f = await fixture(); + const capabilities = await readIndexCapabilities(f.root, { ...f.metadata, graphPath }); + expect(capabilities?.vectorSearch).toMatchObject({ status: 'unavailable' }); + }, + ); + + it('does not report vectors as ready through a symlinked graph generation', async () => { + const f = await fixture(); + await fs.rename(f.graph, f.graph + '.foreign'); + await fs.symlink(f.graph + '.foreign', f.graph); + expect((await readIndexCapabilities(f.root, f.metadata))?.vectorSearch).toMatchObject({ + status: 'unavailable', + }); + }); + + it('advances two staged graphs with a small historical reference and no vector writes', async () => { + const f = await fixture(); + const before = await fs.stat(f.vectors); + for (let i = 0; i < 2; i++) { + const prior = await loadMeta(f.root); + if (!prior) throw new Error('Missing metadata'); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + prior, + '1'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toMatchObject({ + identity, + artifactIdentity, + graphGeneration: generation, + preservedForGraph: receipt.id, + }); + expect( + selectVectorGeneration(f.root, published, path.join(f.root, receipt.graphFile), 10000), + ).toMatchObject({ kind: 'ready', path: f.vectors, coverage: 'preserved' }); + expect(await readIndexCapabilities(f.root, published)).toMatchObject({ + vectorSearch: { + status: 'exact-scan', + coverage: 'preserved', + sourceGraphGeneration: generation, + }, + }); + expect(await readEmbeddingJobProgress(f.root, published)).toEqual({ + phase: 'preserved', + chunks: 3, + }); + } + const after = await fs.stat(f.vectors); + expect([after.ino, after.size, after.mtimeMs]).toEqual([ + before.ino, + before.size, + before.mtimeMs, + ]); + expect(await fs.readFile(f.vectors, 'utf8')).toBe('untouched vector artifact'); + }); + + it('binds preservation before invalidating an in-place graph and retains its inode', async () => { + const f = await fixture(); + const inode = (await fs.stat(f.graph)).ino; + const dirty = await beginInPlaceAnalysis( + f.root, + { + ...f.metadata, + incrementalInProgress: { startedAt: 1, toWriteCount: 1 }, + }, + f.actions.saveMetadata, + ); + expect(selectVectorGeneration(f.root, dirty, f.graph, 10000)).toMatchObject({ + kind: 'unavailable', + }); + const attemptId = dirty.incrementalInProgress?.attemptId; + if (!attemptId) throw new Error('Missing attempt'); + const receipt = await prepareAnalysisPublication( + f.root, + f.graph, + { + ...dirty, + incrementalInProgress: undefined, + }, + '2'.repeat(64), + { mode: 'in-place', attemptId }, + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toMatchObject({ + graphGeneration: generation, + preservedForGraph: receipt.id, + }); + expect(selectVectorGeneration(f.root, published, f.graph, 10000)).toMatchObject({ + kind: 'ready', + coverage: 'preserved', + }); + expect((await fs.stat(f.graph)).ino).toBe(inode); + }); + + it('does not turn an unrelated generation into preserved coverage', async () => { + const f = await fixture(); + const vector = f.metadata.vectorGeneration; + if (!vector) throw new Error('Missing vector fixture'); + const invalid = { + ...f.metadata, + vectorGeneration: { ...vector, graphGeneration: 'f'.repeat(32) }, + }; + await saveMeta(f.root, invalid); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + invalid, + '3'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect( + selectVectorGeneration(f.root, published, path.join(f.root, receipt.graphFile), 10000), + ).toMatchObject({ kind: 'unavailable', reason: 'vector-generation-mismatch' }); + }); + + it('reports a rebased current artifact by artifact identity, not the job directory', async () => { + const f = await fixture(); + expect(await readIndexCapabilities(f.root, f.metadata)).toMatchObject({ + vectorSearch: { + status: 'exact-scan', + coverage: 'current', + sourceGraphGeneration: generation, + }, + }); + }); + + it('gates preserved coverage while a job is pending and rejects a foreign preservation target', async () => { + const f = await fixture(); + const receipt = await prepareAnalysisPublication( + f.root, + await f.stage(), + f.metadata, + '4'.repeat(64), + ); + const published = await publishPreparedAnalysis(f.root, f.actions); + if (!published) throw new Error('Missing publication'); + const graph = path.join(f.root, receipt.graphFile); + expect( + selectVectorGeneration( + f.root, + { ...published, embeddingJob: { version: 1, identity, graphGeneration: receipt.id } }, + graph, + 10000, + ), + ).toEqual({ kind: 'unavailable', reason: 'embedding-pending' }); + expect( + selectVectorGeneration( + f.root, + { + ...published, + graphGeneration: { version: 1, id: '9'.repeat(32), coveredContent: '4'.repeat(64) }, + }, + graph, + 10000, + ), + ).toEqual({ kind: 'unavailable', reason: 'vector-generation-mismatch' }); + }); + + it('forwards preserved progress without inventing current selection totals or leaking identifiers', () => { + const projected = publicOpsProgress({ + phase: 'graph-ready', + percent: 100, + message: 'Graph ready', + embeddingProgress: { + phase: 'preserved', + chunks: 3, + artifactIdentity, + sourcePath: '/private', + }, + } as AnalyzeJobProgress); + expect(projected.embeddingProgress).toEqual({ phase: 'preserved', chunks: 3 }); + }); +}); + +describe('legacy-only metadata publication', () => { + it('does not hide corrupt canonical metadata behind an older legacy mirror', async () => { + const f = await fixture(); + await fs.writeFile(path.join(f.root, 'gitnexus.json'), '{broken'); + await expect( + prepareAnalysisPublication(f.root, await f.stage(), f.metadata, '7'.repeat(64)), + ).rejects.toThrow(); + await expect(fs.stat(path.join(f.root, 'analysis-publication.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('binds the legacy metadata receipt and rejects a superseding legacy writer', async () => { + const f = await fixture(); + await fs.unlink(path.join(f.root, 'gitnexus.json')); + await prepareAnalysisPublication(f.root, await f.stage(), f.metadata, '5'.repeat(64)); + await fs.writeFile( + path.join(f.root, 'meta.json'), + JSON.stringify({ ...f.metadata, lastCommit: 'newer' }), + ); + await expect(publishPreparedAnalysis(f.root, f.actions)).rejects.toThrow('superseded'); + expect((await loadMeta(f.root))?.lastCommit).toBe('newer'); + }); + it('accepts an unchanged legacy receipt without filling in unknown embedding provenance', async () => { + const f = await fixture(); + const legacy = { + ...f.metadata, + graphGeneration: undefined, + vectorGeneration: undefined, + embeddingDims: 4, + }; + await saveMeta(f.root, legacy); + await fs.unlink(path.join(f.root, 'gitnexus.json')); + await prepareAnalysisPublication(f.root, await f.stage(), legacy, '6'.repeat(64)); + const published = await publishPreparedAnalysis(f.root, f.actions); + expect(published?.vectorGeneration).toBeUndefined(); + expect(published?.embeddingDims).toBe(4); + }); +}); diff --git a/gitnexus/test/unit/vector-generation.test.ts b/gitnexus/test/unit/vector-generation.test.ts new file mode 100644 index 000000000..6f6ec092c --- /dev/null +++ b/gitnexus/test/unit/vector-generation.test.ts @@ -0,0 +1,169 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + assertOwnedVectorGeneration, + embeddingGenerationDirectory, + selectVectorGeneration, +} from '../../src/storage/vector-generation.js'; +import type { RepoMeta } from '../../src/storage/repo-meta.js'; + +const generation = 'a'.repeat(32); +const identity = 'b'.repeat(64); +const meta = { + repoPath: '/repo', + lastCommit: 'abc', + indexedAt: 'now', + graphPath: `lbug.generation.${generation}`, + graphGeneration: { version: 1, id: generation, coveredContent: 'c'.repeat(64) }, + vectorGeneration: { + version: 1, + identity, + graphGeneration: generation, + vectorIdentity: 'd'.repeat(64), + inputIdentity: 'e'.repeat(64), + dimensions: 4, + nodes: 1, + chunks: 2, + indexKind: 'exact', + }, +} satisfies RepoMeta; +const select = (metadata: RepoMeta = meta, graphPath = path.join('/slot', meta.graphPath)) => + selectVectorGeneration('/slot', metadata, graphPath, 10000); +const directories: string[] = []; +afterEach(async () => { + for (const directory of directories.splice(0)) + await fs.rm(directory, { recursive: true, force: true }); +}); + +describe('artifact identity and pending-job routing', () => { + it('routes a rebased artifact by its storage identity while retaining the graph-bound job identity', () => { + const artifactIdentity = 'f'.repeat(64); + const metadata = { ...meta, vectorGeneration: { ...meta.vectorGeneration, artifactIdentity } }; + expect(select(metadata)).toEqual({ + kind: 'ready', + path: path.join('/slot', 'embeddings', artifactIdentity, 'vectors'), + generation: metadata.vectorGeneration, + }); + expect( + select({ + ...metadata, + vectorGeneration: { ...metadata.vectorGeneration, artifactIdentity: '../foreign' }, + }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it('gates a pending job even when its identity equals the last completed pointer', () => { + expect( + select({ ...meta, embeddingJob: { version: 1, identity, graphGeneration: generation } }), + ).toEqual({ + kind: 'unavailable', + reason: 'embedding-pending', + }); + }); +}); + +describe('vector generation routing', () => { + it('keeps preserved monolithic vectors readable when a generation was published but embedding was capped', () => { + const preserved: RepoMeta = { + ...meta, + vectorGeneration: undefined, + legacyVectorSource: 'f'.repeat(64), + embeddingDims: 4, + stats: { embeddings: 2 }, + }; + expect(select(preserved)).toEqual({ kind: 'legacy' }); + for (const changed of [ + { legacyVectorSource: '../foreign' }, + { stats: { embeddings: 0 } }, + { embeddingDims: 0 }, + { embeddingJob: { version: 1 as const, identity, graphGeneration: generation } }, + ]) + expect(select({ ...preserved, ...changed })).toMatchObject({ kind: 'unavailable' }); + expect(select(preserved, '/slot/foreign')).toMatchObject({ kind: 'unavailable' }); + }); + it.each([ + { incrementalInProgress: { startedAt: 1, toWriteCount: 1 } }, + { graphWriteCollapsed: { expected: 100, persisted: 1 } }, + ])('gates incomplete legacy and generation graphs before vector access: %o', (incomplete) => { + for (const base of [meta, { repoPath: '/repo', lastCommit: '', indexedAt: '' }]) + expect(select({ ...base, ...incomplete })).toEqual({ + kind: 'unavailable', + reason: 'graph-generation-unverified', + }); + }); + it('preserves the legacy reader and routes a complete matching artifact', () => { + expect(select({ repoPath: '/repo', lastCommit: '', indexedAt: '' })).toEqual({ + kind: 'legacy', + }); + expect(select()).toEqual({ + kind: 'ready', + path: path.join('/slot', 'embeddings', identity, 'vectors'), + generation: meta.vectorGeneration, + }); + }); + it('gates pending, foreign, missing and oversized exact artifacts', () => { + expect( + select({ + ...meta, + vectorGeneration: undefined, + embeddingJob: { version: 1, identity, graphGeneration: generation }, + }), + ).toMatchObject({ kind: 'unavailable', reason: 'embedding-pending' }); + expect(select({ ...meta, graphGeneration: undefined })).toMatchObject({ kind: 'unavailable' }); + expect(select(meta, '/slot/lbug')).toMatchObject({ kind: 'unavailable' }); + expect( + select({ + ...meta, + vectorGeneration: { ...meta.vectorGeneration, graphGeneration: 'f'.repeat(32) }, + }), + ).toMatchObject({ kind: 'unavailable' }); + expect( + select({ ...meta, vectorGeneration: { ...meta.vectorGeneration, chunks: 10001 } }), + ).toMatchObject({ kind: 'unavailable', reason: 'vector-index-pending' }); + expect( + select({ + ...meta, + embeddingJob: { version: 1, identity: 'f'.repeat(64), graphGeneration: generation }, + }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it('admits completed coverage with excluded inputs but rejects missing healthy chunks', () => { + const vectorGeneration = { + ...meta.vectorGeneration, + nodes: 3, + chunks: 2, + excludedNodes: 1, + exclusionReason: 'input-rejected' as const, + }; + expect(select({ ...meta, vectorGeneration })).toMatchObject({ kind: 'ready' }); + expect(select({ ...meta, vectorGeneration: { ...vectorGeneration, chunks: 1 } })).toMatchObject( + { + kind: 'unavailable', + reason: 'vector-generation-mismatch', + }, + ); + }); + it.each([NaN, -1, 0.5, Infinity])('rejects invalid coverage %s', (chunks) => { + expect( + select({ ...meta, vectorGeneration: { ...meta.vectorGeneration, chunks } }), + ).toMatchObject({ kind: 'unavailable' }); + }); + it('rejects traversal and every symlink boundary', async () => { + expect(() => embeddingGenerationDirectory('/slot', '../foreign')).toThrow('identity'); + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-vector-path-')); + directories.push(root); + const job = embeddingGenerationDirectory(root, identity); + await fs.mkdir(job, { recursive: true }); + const file = path.join(job, 'vectors'); + await fs.writeFile(file, 'artifact'); + expect(await assertOwnedVectorGeneration(root, identity)).toBe(file); + await fs.rename(file, file + '.old'); + await fs.symlink(file + '.old', file); + await expect(assertOwnedVectorGeneration(root, identity)).rejects.toThrow('regular file'); + await fs.unlink(file); + await fs.rename(job, job + '.old'); + await fs.symlink(job + '.old', job, 'dir'); + await expect(assertOwnedVectorGeneration(root, identity)).rejects.toThrow('directory'); + }); +}); diff --git a/gitnexus/test/unit/watch-cancellation.test.ts b/gitnexus/test/unit/watch-cancellation.test.ts new file mode 100644 index 000000000..a0a390034 --- /dev/null +++ b/gitnexus/test/unit/watch-cancellation.test.ts @@ -0,0 +1,84 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ analyze: vi.fn(), mayEmbed: vi.fn(async () => false) })); +vi.mock('../../src/cli/analyze.js', () => ({ + ensureHeap: vi.fn(async () => false), + usesManualAnalysisHeap: () => false, +})); +vi.mock('../../src/core/run-analyze.js', () => ({ + runFullAnalysis: mocks.analyze, + analyzeFailureMayHaveMutatedLiveIndex: () => false, +})); +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: mocks.mayEmbed, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: () => ({ mainHeapMb: 512 }), +})); +import { watchCommandWithRunnerIdentity } from '../../src/cli/analyze-watch.js'; +import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + mocks.analyze.mockReset(); + mocks.mayEmbed.mockReset(); + mocks.mayEmbed.mockResolvedValue(false); +}); + +it.each(['preflight', 'analysis'] as const)( + 'consumes supervisor cancellation during %s and waits for active work', + async (phase) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-cancel-')); + const repo = path.join(root, 'repo'); + await fs.mkdir(repo); + execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: repo, stdio: 'pipe' }); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.spyOn(process, 'send').mockImplementation(() => true); + const listeners = process.listeners('message'); + let finish!: () => void; + const held = new Promise((resolve) => { + finish = resolve; + }); + if (phase === 'preflight') + mocks.mayEmbed.mockImplementationOnce(async () => { + await held; + return false; + }); + mocks.analyze.mockImplementation(async () => { + await held; + return { alreadyUpToDate: true }; + }); + let settled = false; + const run = watchCommandWithRunnerIdentity( + { schemaVersion: 4 } as AnalyzerRunnerIdentity, + repo, + ).finally(() => { + settled = true; + }); + try { + await vi.waitFor(() => + expect(phase === 'preflight' ? mocks.mayEmbed : mocks.analyze).toHaveBeenCalled(), + ); + const control = process + .listeners('message') + .find((listener) => !listeners.includes(listener)); + expect(control).toBeDefined(); + control!({ type: 'cancel' }, undefined); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + finish(); + await run; + expect(process.listeners('message')).not.toContain(control); + expect(mocks.analyze).toHaveBeenCalledTimes(phase === 'preflight' ? 0 : 1); + } finally { + finish(); + await run; + await fs.rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/gitnexus/test/unit/watch-external-config.test.ts b/gitnexus/test/unit/watch-external-config.test.ts index 8a4466d20..7808c784b 100644 --- a/gitnexus/test/unit/watch-external-config.test.ts +++ b/gitnexus/test/unit/watch-external-config.test.ts @@ -39,7 +39,10 @@ vi.mock('../../src/core/run-analyze.js', () => ({ return { alreadyUpToDate: true }; }), })); -vi.mock('../../src/cli/analyze.js', () => ({ ensureHeap: async () => false })); +vi.mock('../../src/cli/analyze.js', () => ({ + ensureHeap: async () => false, + usesManualAnalysisHeap: () => false, +})); vi.mock('../../src/storage/storage-resolver.js', async (importActual) => ({ ...(await importActual()), requireStoragePath: async () => undefined, diff --git a/gitnexus/test/unit/watch-paths.test.ts b/gitnexus/test/unit/watch-paths.test.ts index 0a019d6bb..4effbfdd1 100644 --- a/gitnexus/test/unit/watch-paths.test.ts +++ b/gitnexus/test/unit/watch-paths.test.ts @@ -5,6 +5,16 @@ import path from 'node:path'; import { createWatchIgnorePredicate } from '../../src/config/ignore-service.js'; import { isRelevantWatchPath, resolveWatchOptions } from '../../src/cli/analyze-watch.js'; import * as git from '../../src/storage/git.js'; +import { + analysisMayEmbed, + resolveHostAnalysisResources, +} from '../../src/core/analysis-host-resources.js'; + +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: vi.fn(async () => false), + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: vi.fn(() => ({ mainHeapMb: 512 })), +})); vi.mock('../../src/storage/git.js', () => ({ getCoreExcludesFilePath: vi.fn(), @@ -24,6 +34,25 @@ afterEach(async () => { }); describe('watch path selection', () => { + it('admits the requested family budget and pending embeddings for watch refreshes', async () => { + vi.mocked(analysisMayEmbed).mockResolvedValueOnce(true); + const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; + const result = await resolveWatchOptions( + repoPath, + { resourceBudget: '4096', workers: '2', branch: 'feature' }, + baseline, + ); + expect(analysisMayEmbed).toHaveBeenLastCalledWith(repoPath, { branch: 'feature' }); + expect(resolveHostAnalysisResources).toHaveBeenLastCalledWith( + expect.objectContaining({ + totalMb: 4096, + workers: 2, + includeEmbeddings: true, + mainHeapMb: expect.any(Number), + }), + ); + expect(result).toMatchObject({ graphPublication: 'auto', resources: { mainHeapMb: 512 } }); + }); it('forwards and removes route configuration on subsequent refreshes', async () => { const aliases = [{ importPath: 'example.com/core', type: 'Engine', framework: 'gin' }]; const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; diff --git a/gitnexus/test/unit/watch-supervisor-reservation.test.ts b/gitnexus/test/unit/watch-supervisor-reservation.test.ts new file mode 100644 index 000000000..557b2263a --- /dev/null +++ b/gitnexus/test/unit/watch-supervisor-reservation.test.ts @@ -0,0 +1,106 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { getHeapStatistics } from 'node:v8'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + resources: vi.fn((_request: { supervisorBytes?: number }) => ({ + mainHeapMb: 512, + supervisorBytes: 0, + })), + heap: vi.fn(async () => true), +})); +vi.mock('../../src/cli/analyze.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ensureHeap: mocks.heap, usesManualAnalysisHeap: actual.usesManualAnalysisHeap }; +}); +vi.mock('../../src/storage/storage-resolver.js', () => ({ + ANALYZE_STORAGE_REQUIREMENTS: {}, + requireStoragePath: vi.fn(async () => { + throw new Error('stop after admission'); + }), +})); +vi.mock('../../src/core/analysis-host-resources.js', () => ({ + analysisMayEmbed: async () => false, + analysisSupervisorReservation: () => 192 * 1024 ** 2, + resolveHostAnalysisResources: mocks.resources, +})); +import { + resolveWatchOptions, + watchCommandWithRunnerIdentity, +} from '../../src/cli/analyze-watch.js'; +import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; + +const directories: string[] = []; +beforeEach(() => { + vi.stubEnv('GITNEXUS_MEMORY', ''); + vi.stubEnv('GITNEXUS_ANALYSIS_MANUAL_HEAP', ''); + vi.stubEnv('NODE_OPTIONS', ''); +}); +afterEach(async () => { + process.exitCode = undefined; + vi.unstubAllEnvs(); + vi.clearAllMocks(); + await Promise.all( + directories.splice(0).map((directory) => fs.rm(directory, { recursive: true })), + ); +}); + +it.each([undefined, String(320 * 1024 ** 2)])( + 'accounts for the supervisor before heap respawn with inherited reservation %s', + async (inherited) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); + directories.push(directory); + await fs.mkdir(path.join(directory, '.git')); + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', inherited); + await watchCommandWithRunnerIdentity({} as AnalyzerRunnerIdentity, directory); + expect(mocks.resources).toHaveBeenCalledWith( + expect.objectContaining({ + supervisorBytes: inherited === undefined ? 192 * 1024 ** 2 : Number(inherited), + }), + ); + expect(mocks.heap).toHaveBeenCalledOnce(); + }, +); + +it.each([ + [undefined, undefined], + [String(320 * 1024 ** 2), undefined], + [undefined, '768'], + [String(320 * 1024 ** 2), '768'], +])( + 'retains supervisor %s and explicit heap %s across watch refreshes', + async (inherited, memoryBudget) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-supervisor-')); + directories.push(directory); + vi.stubEnv('GITNEXUS_ANALYSIS_SUPERVISOR_BYTES', inherited); + vi.stubEnv('GITNEXUS_RC', ''); + const baseline = { maxFileSize: undefined, workerTimeout: undefined, verbose: undefined }; + await resolveWatchOptions(directory, { memoryBudget }, baseline); + await resolveWatchOptions(directory, { memoryBudget }, baseline); + expect(mocks.resources).toHaveBeenCalledTimes(2); + for (const [request] of mocks.resources.mock.calls) + expect(request).toMatchObject({ + supervisorBytes: inherited === undefined ? 192 * 1024 ** 2 : Number(inherited), + mainHeapMb: + memoryBudget === undefined + ? Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2) + : Number(memoryBudget), + }); + }, +); + +it('honors GITNEXUS_MEMORY=off in the actual watch admission path', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-watch-manual-')); + directories.push(directory); + await fs.mkdir(path.join(directory, '.git')); + vi.stubEnv('GITNEXUS_MEMORY', 'off'); + await watchCommandWithRunnerIdentity({} as AnalyzerRunnerIdentity, directory); + expect(mocks.heap).not.toHaveBeenCalled(); + expect(mocks.resources).toHaveBeenCalledWith( + expect.objectContaining({ + mainHeapMb: Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2), + }), + ); +}); diff --git a/gitnexus/test/unit/worker-pool-resource-limits.test.ts b/gitnexus/test/unit/worker-pool-resource-limits.test.ts index 94ea73346..6a8fdfc20 100644 --- a/gitnexus/test/unit/worker-pool-resource-limits.test.ts +++ b/gitnexus/test/unit/worker-pool-resource-limits.test.ts @@ -163,4 +163,35 @@ describe('resolveWorkerHeapCapMb (#2649 per-worker heap cap)', () => { fs.rmSync(tempDir, { recursive: true, force: true }); } }); + + it('uses a per-run heap allocation without changing later pools or environment overrides', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-worker-run-limits-')); + const workerPath = path.join(tempDir, 'fake-worker.js'); + fs.writeFileSync(workerPath, '// fake worker'); + process.env.GITNEXUS_WORKER_HEAP_MB = '256'; + try { + const { createWorkerPool } = await import('../../src/core/ingestion/workers/worker-pool.js'); + const bounded = createWorkerPool(pathToFileURL(workerPath), 1, { heapLimitMb: 768 }); + await bounded.terminate(); + const legacy = createWorkerPool(pathToFileURL(workerPath), 1); + await legacy.terminate(); + expect(workerCtorOptions).toEqual([ + expect.objectContaining({ + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: 768 }, + }), + expect.objectContaining({ + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: 256 }, + }), + ]); + expect(process.env.GITNEXUS_WORKER_HEAP_MB).toBe('256'); + for (const heapLimitMb of [0, 511, 512.5, NaN, Infinity]) { + expect(() => createWorkerPool(pathToFileURL(workerPath), 1, { heapLimitMb })).toThrow( + 'heap allocation', + ); + } + expect(workerCtorOptions).toHaveLength(2); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts index 29da7f18f..5ef639491 100644 --- a/gitnexus/test/unit/worker-pool-timeout-retire.test.ts +++ b/gitnexus/test/unit/worker-pool-timeout-retire.test.ts @@ -94,6 +94,212 @@ afterEach(() => { }); describe('worker pool timeout retirement', () => { + it('retains a live worker when its shutdown termination request rejects', async () => { + let live!: TimeoutThenHealthyWorker; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + workerFactory: () => { + live = new TimeoutThenHealthyWorker(); + live.terminate = async () => { + live.terminateCalls++; + throw new Error('termination rejected'); + }; + return live as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(live.terminateCalls).toBe(1); + live.emit('exit', 0); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + live.emit('exit', 0); + await pool.terminate(); + } + }); + it('retains a rejected termination reservation until physical exit, then releases it', async () => { + let broken: TimeoutThenHealthyWorker | undefined; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + shutdownDrainMs: 20, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id === 0) { + broken = worker; + worker.postMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'sub-batch') + queueMicrotask(() => worker.emit('error', new Error('worker failed'))); + }; + worker.terminate = async () => { + worker.terminateCalls++; + throw new Error('termination rejected'); + }; + } + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.dispatch([{ path: 'src/error.ts', content: '' }])).rejects.toThrow(); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + broken!.emit('exit', 1); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + broken?.emit('exit', 1); + await pool.terminate().catch(() => {}); + } + }); + + it('keeps an errored retired worker reserved until its physical exit', async () => { + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 20, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + worker.terminate = async () => { + worker.terminateCalls++; + throw new Error('termination rejected'); + }; + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + await expect(pool.dispatch([{ path: 'src/stalled.ts', content: '' }])).rejects.toThrow(); + const retired = TimeoutThenHealthyWorker.instances[0]; + retired.emit('error', new Error('error before exit')); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + retired.emit('exit', 1); + await expect(pool.terminate()).resolves.toBeUndefined(); + } finally { + TimeoutThenHealthyWorker.instances[0]?.emit('exit', 1); + await pool.terminate().catch(() => {}); + } + }); + + it('refuses replacement and phase handoff while a budgeted native worker has not drained', async () => { + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + heapLimitMb: 512, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 20, + workerFactory: () => + new TimeoutThenHealthyWorker() as unknown as import('node:worker_threads').Worker, + }); + try { + await expect( + pool.dispatch([{ path: 'src/stalled.ts', content: 'const x = 1;' }]), + ).rejects.toThrow(); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + expect(TimeoutThenHealthyWorker.instances[0].terminateCalls).toBe(0); + await expect(pool.terminate()).rejects.toThrow('native resource reservation'); + expect(TimeoutThenHealthyWorker.instances[0].terminateCalls).toBe(0); + } finally { + TimeoutThenHealthyWorker.instances[0].emit('message', { type: 'sub-batch-done' }); + await waitFor( + () => TimeoutThenHealthyWorker.instances[0].terminateCalls === 1, + 'Worker did not retire', + ); + await pool.terminate(); + } + }); + + it('replaces a budgeted worker only after its safe point and native termination complete', async () => { + TimeoutThenHealthyWorker.firstWorkerBehavior = 'delayed-safe-return'; + TimeoutThenHealthyWorker.safeReturnDelayMs = 40; + let completeTermination: (() => void) | undefined; + const pool = createWorkerPool(workerUrl, 1, { + requireDrain: true, + subBatchIdleTimeoutMs: 10, + maxTimeoutRetries: 1, + shutdownDrainMs: 500, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id === 0) + worker.terminate = () => { + worker.terminateCalls++; + return new Promise((resolve) => { + completeTermination = () => { + worker.emit('exit', 0); + resolve(0); + }; + }); + }; + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + try { + const result = pool.dispatch([{ path: 'src/delayed.ts', content: '' }]); + await waitFor(() => completeTermination !== undefined, 'Missing safe termination'); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(1); + completeTermination!(); + await expect(result).resolves.toEqual([{ paths: ['src/delayed.ts'] }]); + expect(TimeoutThenHealthyWorker.instances).toHaveLength(2); + } finally { + completeTermination?.(); + await pool.terminate(); + } + }); + + it('replaces a released slot while another slot still awaits physical exit', async () => { + const release: Array<() => void> = []; + const dispatched = new Set(); + const pool = createWorkerPool(workerUrl, 2, { + requireDrain: true, + shutdownDrainMs: 500, + subBatchIdleTimeoutMs: 2000, + workerFactory: () => { + const worker = new TimeoutThenHealthyWorker(); + if (worker.id < 2) { + worker.postMessage = (message: unknown) => { + if ((message as { type?: string })?.type === 'sub-batch') dispatched.add(worker.id); + }; + worker.terminate = () => + new Promise((resolve) => { + release[worker.id] = () => { + worker.emit('exit', 1); + resolve(1); + }; + }); + } + return worker as unknown as import('node:worker_threads').Worker; + }, + }); + const result = pool.dispatch([ + { path: 'a.ts', content: '' }, + { path: 'b.ts', content: '' }, + ]); + try { + await waitFor(() => dispatched.size === 2, 'Both slots did not dispatch'); + TimeoutThenHealthyWorker.instances[0].emit('error', new Error('slot zero failed')); + TimeoutThenHealthyWorker.instances[1].emit('error', new Error('slot one failed')); + await waitFor(() => release.filter(Boolean).length === 2, 'Missing termination requests'); + release[0](); + await waitFor( + () => TimeoutThenHealthyWorker.instances.length === 3, + 'Released slot was blocked by the other slot', + ); + release[1](); + await result; // Failed inputs are quarantined; both replacement slots remain usable. + expect(TimeoutThenHealthyWorker.instances).toHaveLength(4); + const results = await pool.dispatch([ + { path: 'healthy-a.ts', content: '' }, + { path: 'healthy-b.ts', content: '' }, + ]); + expect(results).toEqual( + expect.arrayContaining([{ paths: ['healthy-a.ts'] }, { paths: ['healthy-b.ts'] }]), + ); + } finally { + for (const finish of release) finish?.(); + await result.catch(() => undefined); + await pool.terminate(); + } + }); + it('does not immediately terminate a worker that timed out inside native parsing', async () => { const pool = createWorkerPool(workerUrl, 1, { subBatchIdleTimeoutMs: 20, diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 684046e6e..e29826259 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -1,6 +1,22 @@ import { defineConfig } from 'vitest/config'; import PerfSequencer from './test/helpers/perf-sequencer.js'; +// Keep native tests in exactly one project (TESTING.md). +const RESUMABLE_ANALYSIS_NATIVE_TESTS = [ + 'test/integration/embedding-input-isolation.test.ts', + 'test/integration/embedding-vector-store.test.ts', + 'test/integration/legacy-vector-reuse-index.test.ts', + 'test/integration/legacy-vector-export.test.ts', + 'test/integration/analysis-embedding-flow.test.ts', + 'test/integration/analysis-generation-routing.test.ts', + 'test/integration/analysis-publication-flow.test.ts', + 'test/integration/embedding-job-controller.test.ts', + 'test/integration/embedding-job-restart.test.ts', + 'test/integration/embedding-native-release.test.ts', + 'test/integration/legacy-embedding-controller.test.ts', + 'test/integration/legacy-vector-source.test.ts', +]; + export default defineConfig({ test: { // Shared settings — inherited by all projects via extends: true @@ -78,6 +94,7 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', @@ -196,6 +213,7 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + ...RESUMABLE_ANALYSIS_NATIVE_TESTS, 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts',