mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-14 09:01:33 +00:00
Merge aa3e90e89e into ad7c182243
This commit is contained in:
commit
05b0eb33e9
204 changed files with 27453 additions and 1915 deletions
89
.github/workflows/ci-tests.yml
vendored
89
.github/workflows/ci-tests.yml
vendored
|
|
@ -7,6 +7,95 @@ permissions:
|
|||
contents: read
|
||||
|
||||
jobs:
|
||||
analyze-resume-benchmark:
|
||||
name: analyze / resumable performance (${{ matrix.resources }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
resources: [explicit, default]
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Pin comparison revision
|
||||
id: baseline
|
||||
shell: bash
|
||||
env:
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
if [[ -n "$PR_BASE_SHA" ]]; then
|
||||
comparison_sha="$PR_BASE_SHA"
|
||||
else
|
||||
comparison_sha="$(git rev-parse --verify HEAD^1)"
|
||||
fi
|
||||
printf 'sha=%s\n' "$comparison_sha" >> "$GITHUB_OUTPUT"
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
build: 'true'
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ steps.baseline.outputs.sha }}
|
||||
path: baseline
|
||||
persist-credentials: false
|
||||
- name: Build baseline runtime
|
||||
working-directory: baseline/gitnexus
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
npx tsx scripts/ensure-fts.ts
|
||||
- name: Install candidate search extensions
|
||||
working-directory: gitnexus
|
||||
run: npx tsx scripts/ensure-fts.ts
|
||||
- name: Profile fresh HTTP analysis separately from acceptance timings
|
||||
if: matrix.resources == 'explicit'
|
||||
working-directory: gitnexus
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/analyze-profile"
|
||||
node bench/analyze-resume/measure.mjs --cpu-profile --allow-dependency-changes \
|
||||
--scenarios http-fresh --pairs 1 --warmups 0 \
|
||||
--baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \
|
||||
--output "$RUNNER_TEMP/analyze-profile/results.json"
|
||||
- name: Retain diagnostic CPU profiles before the full comparison
|
||||
if: always() && matrix.resources == 'explicit'
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: analyze-resume-cpu-profile-${{ matrix.resources }}
|
||||
path: |
|
||||
${{ runner.temp }}/analyze-profile/**/*.cpuprofile
|
||||
${{ runner.temp }}/analyze-profile/**/*.json
|
||||
${{ runner.temp }}/analyze-profile/**/*.log
|
||||
retention-days: 7
|
||||
- name: Check paired analyze and resume performance
|
||||
working-directory: gitnexus
|
||||
shell: bash
|
||||
env:
|
||||
RESOURCE_PROFILE: ${{ matrix.resources }}
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/analyze-resume"
|
||||
resource_args=()
|
||||
if [[ "$RESOURCE_PROFILE" == default ]]; then
|
||||
resource_args+=(--default-resources)
|
||||
fi
|
||||
node bench/analyze-resume/measure.mjs --validate-gate
|
||||
node bench/analyze-resume/measure.mjs --check --allow-dependency-changes \
|
||||
--baseline "$GITHUB_WORKSPACE/baseline/gitnexus/dist/cli/index.js" \
|
||||
--output "$RUNNER_TEMP/analyze-resume/results.json" \
|
||||
"${resource_args[@]}" 2>&1 | tee "$RUNNER_TEMP/analyze-resume/run.log"
|
||||
- name: Retain paired measurements and analyzer logs
|
||||
if: always()
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: analyze-resume-performance-${{ matrix.resources }}
|
||||
path: |
|
||||
${{ runner.temp }}/analyze-resume/*.json
|
||||
${{ runner.temp }}/analyze-resume/*.log
|
||||
${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.json
|
||||
${{ runner.temp }}/analyze-resume/analyze-pairs-*/*/*.log
|
||||
retention-days: 30
|
||||
|
||||
lexical-lookup-benchmark:
|
||||
name: indexing / lexical lookup and cache interning
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -640,25 +640,31 @@ Inheritance is captured by the `@reference.inherits` tag and emitted by the scop
|
|||
|
||||
```
|
||||
CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks)
|
||||
1. Early exit if lastCommit == HEAD (unless --force) [0%]
|
||||
2. Cache existing embeddings from prior index [0%]
|
||||
3. runPipelineFromRepo() → KnowledgeGraph [0-60%]
|
||||
4. Clean up legacy KuzuDB files [60%]
|
||||
5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%]
|
||||
6. Create FTS indexes (File, Function, Class, Method...) [85-90%]
|
||||
7. Restore cached embeddings (batch insert) [88%]
|
||||
8. Generate new embeddings if --embeddings [90-98%]
|
||||
9. Save metadata + register repo + update .gitignore [98-100%]
|
||||
10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%]
|
||||
1. Admit phase resources and recover any prepared graph publication.
|
||||
2. Reuse unchanged graph/parser work where its receipts still match.
|
||||
3. runPipelineFromRepo() → KnowledgeGraph → bounded graph write → FTS.
|
||||
4. Prepare the owned graph and publication receipt; drain native resources.
|
||||
5. Supervisor waits for graph-process exit, verifies and publishes metadata.
|
||||
6. If embeddings are requested/pending, open the resumable embedding job.
|
||||
7. Reuse verified vectors, checkpoint bounded units, then seal/publish vectors.
|
||||
8. Register the repo and generate AI context files.
|
||||
|
||||
Graph-only legacy slots keep their existing path until separate publication is
|
||||
needed. A ready graph remains queryable while an embedding job is pending;
|
||||
semantic search reports degradation until a compatible sealed artifact is ready.
|
||||
```
|
||||
|
||||
**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`.
|
||||
**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, resumable), `--skipGit`, `--noStats`.
|
||||
|
||||
## Storage
|
||||
|
||||
```
|
||||
<repo>/.gitnexus/
|
||||
├── lbug # LadybugDB database
|
||||
├── lbug # Legacy monolithic database, until first separate publication
|
||||
├── lbug.generation.<id> # Owned private graph, selected by graphPath in metadata
|
||||
├── analysis-publication.json # Recoverable prepared graph receipt
|
||||
├── embeddings/<identity>/ # Job journal, manifest and separate vector database
|
||||
├── embeddings/spaces/ # Model-space ownership/reuse receipts
|
||||
├── lbug.wal # Write-ahead log
|
||||
├── lbug.shadow # Shadow sidecar (checkpoint staging)
|
||||
├── lbug.lock # Single-writer lock
|
||||
|
|
@ -690,7 +696,11 @@ the read-only open is retried — see `sidecar-recovery.ts`
|
|||
(`isReadOnlyCheckpointInProgressError`) and the
|
||||
`lbug-interrupted-checkpoint-recovery` integration test.
|
||||
|
||||
Managed by `repo-manager.ts`.
|
||||
Managed by `repo-manager.ts`. Publication reclaims unreferenced private graphs
|
||||
and job artifacts under the slot lock, preserving current jobs, model spaces,
|
||||
legacy reuse sources and prepared publications. Busy reader files are retried on
|
||||
a later publication. See [MIGRATION.md](MIGRATION.md#resumable-analysis-and-separate-vector-storage-3598)
|
||||
before mixing binary versions or rolling back.
|
||||
|
||||
### Automatic parse-cache identity
|
||||
|
||||
|
|
|
|||
|
|
@ -44,13 +44,13 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m
|
|||
|
||||
- **Trigger:** Semantic search quality drops; `stats.embeddings` in the index metadata (`gitnexus.json` / legacy `meta.json`) is 0 after refresh.
|
||||
- **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected.
|
||||
- **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings.
|
||||
- **Why:** Plain `analyze` preserves prior vectors in their artifact (or restores the legacy in-graph cache); ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings.
|
||||
|
||||
### Analyze finishes but embeddings are incomplete (partial embedding index)
|
||||
|
||||
- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["embedding-checkpoint-pending"]` (or the human-readable "Index incomplete reasons" line); `stats.embeddings` is honest and **non-zero**, and the preceding analyze log showed a `Warning: N node(s) lost their embeddings to embedding-endpoint failures` line (#2790).
|
||||
- **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it.
|
||||
- **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal.
|
||||
- **Trigger:** `gitnexus status --json` reports `embedding-job-pending` (or the legacy `embedding-checkpoint-pending`), or an embedding failure follows graph publication.
|
||||
- **Do:** Re-run plain `gitnexus analyze`; retained work resumes without `--embeddings`. Inspect `embeddingProgress` for completed, reused, failed and excluded nodes. `--force` rebuilds the graph and retains compatible vectors for reuse; `--drop-embeddings` deliberately abandons embedding work.
|
||||
- **Why:** Graph and vector publication are separate. The graph is usable while a job is pending; semantic search reports degradation until a sealed compatible artifact is published. Deterministic input rejections are retried across at most three invocations, then excluded with a recorded reason. Provider outages remain resumable. Do not delete job artifacts to clear a status warning: they contain paid vectors.
|
||||
|
||||
### Scope extraction is incomplete
|
||||
|
||||
|
|
|
|||
52
MIGRATION.md
52
MIGRATION.md
|
|
@ -1,5 +1,57 @@
|
|||
# Migration Guide
|
||||
|
||||
## Resumable analysis and separate vector storage (#3598)
|
||||
|
||||
Analysis with embeddings publishes a private `lbug.generation.<id>` graph and
|
||||
stores resumable vector artifacts under `embeddings/<identity>/`. Metadata names
|
||||
the active graph and sealed vector generation. Once a slot uses this layout,
|
||||
plain analysis, watch and hooks continue using it. Pending embedding work resumes
|
||||
on the next plain `gitnexus analyze`; `--force` rebuilds the graph while retaining
|
||||
compatible vectors for reuse. `--drop-embeddings` explicitly abandons embeddings.
|
||||
|
||||
Upgrade every CLI, MCP/serve process and installed hook that accesses the slot.
|
||||
Restart long-running clients and rerun `gitnexus setup` with the upgraded binary.
|
||||
Older versions do not understand the private graph pointer or separate vectors.
|
||||
Do not run an older analyzer against the migrated slot.
|
||||
|
||||
Before upgrading, stop readers and writers and back up the complete storage
|
||||
slot, including metadata, graph files, `embeddings/`, and any referenced shared
|
||||
store. To roll back, stop clients again and restore that complete backup with
|
||||
the matching older binaries. Renaming a generation file to `lbug` does not move
|
||||
vectors back into it. There is no reverse conversion for separate vector
|
||||
artifacts; without a backup, rebuilding with an older version requires generating
|
||||
embeddings again and can incur provider charges.
|
||||
|
||||
A graph can be ready while embeddings remain pending. Embedding failure after
|
||||
graph publication exits nonzero; the graph remains usable and the next analyze
|
||||
resumes paid work. HTTP clients should read `embeddingProgress` and capability
|
||||
reasons rather than relying on the former terminal SSE `partial` field. After
|
||||
three deterministic input rejections, the job publishes healthy nodes and records
|
||||
`excludedNodes` with reason `input-rejected`; status reports the omission.
|
||||
On Windows, a legacy reader can prevent retirement of `lbug`. Stop that reader
|
||||
and rerun analyze; the prepared publication and vector artifacts are retained.
|
||||
|
||||
Legacy source receipts now include the file's change time as well as its size
|
||||
and modification time. Receipts from earlier previews without that guard are
|
||||
retained but cannot certify reuse. Keep them with the original source when
|
||||
recovering from a backup or re-exporting a verified source; do not fill in a
|
||||
missing guard from the current file, which may have changed since retirement.
|
||||
|
||||
`GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` is no longer accepted by analysis: it asks
|
||||
the native engine to allocate an unbounded pool. Remove this setting from CLI,
|
||||
server and auto-sync environments to use automatic sizing, or set a positive
|
||||
byte limit. An explicit `--resource-budget` must also fit the page-size-scaled
|
||||
COPY floor; increase the budget if admission rejects it.
|
||||
|
||||
If a superseded or unverifiable `analysis-publication.json` blocks recovery,
|
||||
stop every reader and writer and back up the complete storage slot first.
|
||||
Move only that receipt into the backup, outside the slot, then run
|
||||
`gitnexus analyze --force`. Keep the staged graphs and embedding artifacts until
|
||||
the replacement graph and vectors are verified. A valid receipt should normally
|
||||
be recovered by plain analysis; discarding it abandons its prepared graph work.
|
||||
For an interrupted in-place write, keep clients stopped until the forced rebuild
|
||||
completes; removing the receipt does not make the old graph healthy.
|
||||
|
||||
## LadybugDB 0.22.0 storage and extensions
|
||||
|
||||
LadybugDB 0.22.0 reads indexes written by 0.21.1, but a write/checkpoint upgrades
|
||||
|
|
|
|||
27
RUNBOOK.md
27
RUNBOOK.md
|
|
@ -129,13 +129,15 @@ costs; compare complete-source candidates separately from older capped inputs.
|
|||
|
||||
**Important:** If you already had embeddings, a plain `npx gitnexus analyze` **preserves** them (Non-negotiable 5 in [GUARDRAILS.md](GUARDRAILS.md)) — pass `--embeddings` when you also want vectors generated for new or changed nodes, and `--drop-embeddings` only for a deliberate wipe. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none) — but that figure isn't always freshly measured: if a run's embedding-count query can't answer, it carries the previous run's number forward instead of writing a wrong zero. For a certified read, check `capabilities.vectorSearch.status` instead — it reads `unavailable` (never a stale count) whenever GitNexus can't vouch for the live vector index.
|
||||
|
||||
**Partial embedding index (analyze exits 0, but some nodes never got embedded):** A long run against a flaky embedding endpoint can finish successfully while a bounded number of sub-batches still fail. Affected nodes are dropped to zero rows (never left half-written) and recorded as a pending `embeddingCheckpoint`; `npx gitnexus status` then reports `incompleteReasons: ["embedding-checkpoint-pending"]`. Recovery is a plain:
|
||||
**Embedding work interrupted after graph publication:** The graph and keyword index become available before embedding work starts. An embedding failure makes analyze exit nonzero while keeping that graph available and retaining acknowledged embedding work. `npx gitnexus status --json` reports graph freshness separately from embedding progress and vector readiness. Resume with:
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze
|
||||
```
|
||||
|
||||
No `--embeddings` flag needed — a retained checkpoint forces embedding generation for the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it.
|
||||
No `--embeddings` flag is needed for a retained job. `npx gitnexus embeddings sync` also resumes work against an already published graph. Compatible acknowledged units do not call the provider again; a unit interrupted before acknowledgement may replay. A source or embedding-configuration change revalidates reuse, and changing worker counts or memory budgets does not invalidate paid vectors. `--drop-embeddings` deliberately abandons retained vectors. `--force` rebuilds the graph while retaining compatible paid work for reuse.
|
||||
|
||||
Older indexes may instead carry an `embeddingCheckpoint` and report `embedding-checkpoint-pending`. Plain analyze recovers that checkpoint, including any explicitly referenced staging data, before migrating it. Historical vectors without verified provider/model provenance are retained without being relabeled as output from the current model.
|
||||
|
||||
**Collapsed graph write (analyze exits NON-ZERO and says INCOMPLETE):** A run can finish writing metadata while only a fraction of the relationships it produced are readable back from the index — edges collapsing to a small share of what was built, or a `CodeRelation` table that never materialized (which reads as a persisted count of zero). Because the metadata IS written and the DB does hold rows, nothing looks broken: queries answer with missing edges rather than an error, which is a confident empty answer rather than a failure. `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`, the analyze summary prints `Repository indexed INCOMPLETELY` with the expected and persisted counts, and the CLI exits non-zero so automation is not told an unusable index is fine.
|
||||
|
||||
|
|
@ -280,6 +282,20 @@ on a partial result.
|
|||
|
||||
Analyze re-execs Node with a **large old-space heap** when needed (`analyze.ts`). If you still OOM on huge repos, close other processes, avoid `--embeddings` for a first pass, or analyze a smaller path if supported by your workflow.
|
||||
|
||||
`--resource-budget <MiB>` admits the analyzer process family within one memory
|
||||
reservation: the frontend, main heap, parser workers, native graph/vector pools,
|
||||
embedding model, bounded input queue, and headroom. For example:
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze --embeddings --resource-budget 8192 --memory-budget 3000 --workers 2
|
||||
```
|
||||
|
||||
`--memory-budget` still controls only the main V8 heap. Explicit settings that do
|
||||
not fit the family budget fail before work starts. Without a family override,
|
||||
GitNexus derives reservations from available memory and CPUs. The budget controls
|
||||
admission and queue sizes; it is not an operating-system RSS limit. Watch mode
|
||||
accepts the same family budget and drains active work before shutdown.
|
||||
|
||||
`Unable to allocate memory` / `The buffer pool is full` during graph writes
|
||||
refers to LadybugDB's native buffer pool. Increasing `NODE_OPTIONS`,
|
||||
`--memory-budget`, or worker heap limits does not resize that pool. Recognized
|
||||
|
|
@ -294,8 +310,11 @@ GITNEXUS_LBUG_BUFFER_POOL_SIZE=3221225472 npx gitnexus analyze --force
|
|||
|
||||
The override bypasses automatic pool sizing. Budget for the Node heap, parse
|
||||
workers, other native allocations, and the OS alongside it; a larger pool can
|
||||
exhaust the host. `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0` restores LadybugDB's native
|
||||
80%-of-RAM default, not an unlimited supply of memory. If memory is insufficient,
|
||||
exhaust the host. Outside bounded analysis, `GITNEXUS_LBUG_BUFFER_POOL_SIZE=0`
|
||||
selects LadybugDB's native 80%-of-RAM default. Analysis and embedding admission
|
||||
reject zero because that native allocation cannot fit alongside their other
|
||||
reservations. Unset the variable for automatic sizing, or set an explicit byte
|
||||
limit of at least 8 MiB that fits the resource budget. If memory is insufficient,
|
||||
exclude generated/vendor directories or use a larger host.
|
||||
|
||||
A measured graph-write collapse aborts a staged build before publication,
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ gitnexus analyze --max-processes <n> # Process-detection process cap (replaces
|
|||
gitnexus analyze --max-entry-point-candidates <n> # Ranked entry-point pool (default 200; raise when the warning names it)
|
||||
gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots
|
||||
gitnexus analyze --asyncapi-spec ./docs/asyncapi # Resolve broker addresses from AsyncAPI 3.x documents
|
||||
gitnexus analyze --resource-budget 8192 # Strict process-family reservation in MiB
|
||||
gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and --max-old-space-size
|
||||
gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes
|
||||
# (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB)
|
||||
|
|
@ -284,7 +285,7 @@ For local model setup and switching instructions, see [Local Embedding Model](..
|
|||
| `GITNEXUS_CONTENT_RETENTION` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | You need to reduce persisted source text while preserving graph structure. |
|
||||
| `GITNEXUS_SKIP_FTS` | unset | When exactly `1`, skips FTS extension loading and keyword index creation during analyze. Equivalent to `--skip-fts`; a later analyze without either option restores FTS. | Graph-only consumers with their own retrieval, or short-lived indexes that do not need keyword search. |
|
||||
| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold <bytes>`. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. |
|
||||
| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. |
|
||||
| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded analysis requires a positive byte ceiling of at least 8 MiB; zero and malformed values fail preflight. Other readers retain the legacy zero/default policy. During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. |
|
||||
| `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning, and invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. |
|
||||
| `GITNEXUS_ATOMIC_INCREMENTAL` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. | A staged incremental analyze fails (see _Troubleshooting_ in the package README), or you need in-place writes. |
|
||||
| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. |
|
||||
|
|
@ -381,3 +382,31 @@ flowchart TD
|
|||
ConnA -->|"queries"| RepoA
|
||||
ConnB -->|"queries"| RepoB
|
||||
```
|
||||
|
||||
### Analysis resource admission and resumable status
|
||||
|
||||
Automatic analysis retains the RAM-aware main heap and independent parser-worker
|
||||
ceilings. These are virtual limits and may sum above physical RAM. Use
|
||||
`--resource-budget <MiB>` for a strict phase reservation covering the supervisor,
|
||||
main heap, workers, native pools, queues and embedding model. The two phases do
|
||||
not overlap. `--memory-budget` alone changes the main heap without removing parser
|
||||
workers; with a strict total budget, incompatible overrides fail before work.
|
||||
Embedding thread settings are validated only when embeddings are requested or pending.
|
||||
|
||||
`status --json` reports `index.graphStatus` for a checkout,
|
||||
`index.graphIncompleteReasons`, `index.embeddingProgress` and `index.capabilities`.
|
||||
Progress includes its phase, completed/reused/failed nodes and pending units;
|
||||
`excludedNodes` records permanently rejected inputs. Stored metadata separately
|
||||
names `graphGeneration`, `vectorGeneration` and `embeddingJob`.
|
||||
A preserved vector generation may refer to an older graph. Pending jobs report
|
||||
`embedding-job-pending`; a published artifact with omitted inputs reports
|
||||
`embedding-inputs-rejected`. A ready graph does not imply ready semantic search.
|
||||
|
||||
`GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` is an internal inherited reservation, in
|
||||
nonnegative integer bytes, for the waiting parent process. Leave it unset for
|
||||
automatic accounting; a supplied value is included in admission. It does not
|
||||
set the parent's V8 heap limit.
|
||||
|
||||
A local-model job can report "Embedding runtime is busy" while an existing query
|
||||
owns the model. Retry after that query finishes; admission does not replace a
|
||||
sidecar that still has physical requests in flight.
|
||||
|
|
|
|||
|
|
@ -304,10 +304,21 @@ function isDirectChild(parent, child) {
|
|||
}
|
||||
|
||||
// Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a
|
||||
// shared-store checkout slot may read a commit graph in the same store
|
||||
// instead of owning <slot>/lbug. Any other recorded value is ignored.
|
||||
// slot may read an owned private generation; shared checkout slots may also
|
||||
// read a commit graph in the same store. Other recorded values are ignored.
|
||||
function resolveGraphPath(storagePath, metadata) {
|
||||
const own = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) {
|
||||
const graph = path.join(storagePath, recorded);
|
||||
try {
|
||||
const stat = fs.lstatSync(graph);
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) return null;
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') return null;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
const storesRoot = path.resolve(
|
||||
process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'),
|
||||
'stores',
|
||||
|
|
@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) {
|
|||
const root = path.dirname(checkoutsDir);
|
||||
if (path.basename(checkoutsDir) !== 'checkouts') return own;
|
||||
if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own;
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own;
|
||||
const graph = path.resolve(recorded);
|
||||
// Only a published `<commit>-<featureKey>` dir, never `.publish-*` staging.
|
||||
|
|
@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) {
|
|||
const branchDir =
|
||||
branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null;
|
||||
const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath;
|
||||
const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir);
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) return null;
|
||||
return {
|
||||
path: current,
|
||||
storagePath,
|
||||
lbugPath: path.join(indexDir, LBUG_DIRECTORY),
|
||||
metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir),
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) {
|
|||
const indexDir = branchIsIndexed
|
||||
? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch))
|
||||
: storagePath;
|
||||
const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata;
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) continue;
|
||||
if (registeredPath.length > bestLen) {
|
||||
bestLen = registeredPath.length;
|
||||
best = {
|
||||
path: entry.path,
|
||||
storagePath,
|
||||
lbugPath: branchIsIndexed
|
||||
? path.join(indexDir, LBUG_DIRECTORY)
|
||||
: resolveGraphPath(storagePath, ownershipMetadata),
|
||||
metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata,
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -304,10 +304,21 @@ function isDirectChild(parent, child) {
|
|||
}
|
||||
|
||||
// Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a
|
||||
// shared-store checkout slot may read a commit graph in the same store
|
||||
// instead of owning <slot>/lbug. Any other recorded value is ignored.
|
||||
// slot may read an owned private generation; shared checkout slots may also
|
||||
// read a commit graph in the same store. Other recorded values are ignored.
|
||||
function resolveGraphPath(storagePath, metadata) {
|
||||
const own = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) {
|
||||
const graph = path.join(storagePath, recorded);
|
||||
try {
|
||||
const stat = fs.lstatSync(graph);
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) return null;
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') return null;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
const storesRoot = path.resolve(
|
||||
process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'),
|
||||
'stores',
|
||||
|
|
@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) {
|
|||
const root = path.dirname(checkoutsDir);
|
||||
if (path.basename(checkoutsDir) !== 'checkouts') return own;
|
||||
if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own;
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own;
|
||||
const graph = path.resolve(recorded);
|
||||
// Only a published `<commit>-<featureKey>` dir, never `.publish-*` staging.
|
||||
|
|
@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) {
|
|||
const branchDir =
|
||||
branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null;
|
||||
const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath;
|
||||
const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir);
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) return null;
|
||||
return {
|
||||
path: current,
|
||||
storagePath,
|
||||
lbugPath: path.join(indexDir, LBUG_DIRECTORY),
|
||||
metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir),
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) {
|
|||
const indexDir = branchIsIndexed
|
||||
? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch))
|
||||
: storagePath;
|
||||
const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata;
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) continue;
|
||||
if (registeredPath.length > bestLen) {
|
||||
bestLen = registeredPath.length;
|
||||
best = {
|
||||
path: entry.path,
|
||||
storagePath,
|
||||
lbugPath: branchIsIndexed
|
||||
? path.join(indexDir, LBUG_DIRECTORY)
|
||||
: resolveGraphPath(storagePath, ownershipMetadata),
|
||||
metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata,
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -304,10 +304,21 @@ function isDirectChild(parent, child) {
|
|||
}
|
||||
|
||||
// Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a
|
||||
// shared-store checkout slot may read a commit graph in the same store
|
||||
// instead of owning <slot>/lbug. Any other recorded value is ignored.
|
||||
// slot may read an owned private generation; shared checkout slots may also
|
||||
// read a commit graph in the same store. Other recorded values are ignored.
|
||||
function resolveGraphPath(storagePath, metadata) {
|
||||
const own = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) {
|
||||
const graph = path.join(storagePath, recorded);
|
||||
try {
|
||||
const stat = fs.lstatSync(graph);
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) return null;
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') return null;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
const storesRoot = path.resolve(
|
||||
process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'),
|
||||
'stores',
|
||||
|
|
@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) {
|
|||
const root = path.dirname(checkoutsDir);
|
||||
if (path.basename(checkoutsDir) !== 'checkouts') return own;
|
||||
if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own;
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own;
|
||||
const graph = path.resolve(recorded);
|
||||
// Only a published `<commit>-<featureKey>` dir, never `.publish-*` staging.
|
||||
|
|
@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) {
|
|||
const branchDir =
|
||||
branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null;
|
||||
const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath;
|
||||
const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir);
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) return null;
|
||||
return {
|
||||
path: current,
|
||||
storagePath,
|
||||
lbugPath: path.join(indexDir, LBUG_DIRECTORY),
|
||||
metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir),
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) {
|
|||
const indexDir = branchIsIndexed
|
||||
? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch))
|
||||
: storagePath;
|
||||
const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata;
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) continue;
|
||||
if (registeredPath.length > bestLen) {
|
||||
bestLen = registeredPath.length;
|
||||
best = {
|
||||
path: entry.path,
|
||||
storagePath,
|
||||
lbugPath: branchIsIndexed
|
||||
? path.join(indexDir, LBUG_DIRECTORY)
|
||||
: resolveGraphPath(storagePath, ownershipMetadata),
|
||||
metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata,
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -299,6 +299,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl
|
|||
gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768)
|
||||
gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses
|
||||
gitnexus analyze --memory-budget 3000 # Main-thread V8 heap in MB (>= 200); overrides the auto-sizer and any --max-old-space-size pin
|
||||
gitnexus analyze --resource-budget 8192 # Process-family memory reservation in MiB, including workers, native pools and embedding work
|
||||
gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB)
|
||||
gitnexus auto-sync [init|start|restart|stop|status|reset] # Scheduled remote clone/pull + analyze from GITNEXUS_HOME/watch_config.yml
|
||||
gitnexus mcp # Start MCP server (stdio) — serves all indexed repos
|
||||
|
|
@ -778,7 +779,7 @@ Configure the behavior with these environment variables:
|
|||
| `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. |
|
||||
| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. |
|
||||
| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
|
||||
| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores the native 80%-of-RAM default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. |
|
||||
| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | Native buffer-pool ceiling in bytes. Bounded analysis requires a positive integer of at least 8 MiB; zero and malformed values fail preflight. Query readers retain the legacy zero/default policy. |
|
||||
| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). The native engine requires a power of two: any other positive integer is rounded up to the next power of two with a warning. Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. |
|
||||
| `GITNEXUS_ATOMIC_INCREMENTAL` | `0`, `1` | unset (the caller's choice) | `1` stages an incremental analyze into a copy of the index and swaps it in at the end, so a failed run leaves the live index untouched. `0` turns that off even for callers that request it (watch mode, auto-sync); the run logs a line when it overrides one. The cost: a failed in-place write can leave the live index partly written, and the next run then rebuilds it in full. A full rebuild, and an incremental run that escalates to a staged rebuild, still stage. POSIX only. |
|
||||
|
||||
|
|
@ -842,6 +843,22 @@ To set the main-thread heap yourself on a memory-constrained host, pass
|
|||
the auto-sizer and any `--max-old-space-size` pin. It sizes the main thread
|
||||
only; parse workers keep their own caps.
|
||||
|
||||
Use `--resource-budget <MiB>` to admit the complete analyzer process family,
|
||||
including the frontend, main heap, workers, native pools, embedding model, input
|
||||
queue, and headroom. An explicit heap or pool that cannot fit is rejected before
|
||||
work starts. Automatic sizing retains independent RAM-aware main and worker
|
||||
ceilings, whose sum may exceed RAM. Only an explicit resource budget imposes an
|
||||
aggregate phase ceiling; this reservation is
|
||||
not an operating-system RSS limit. `analyze --watch` accepts the same budget.
|
||||
|
||||
Embedding-enabled analysis publishes the graph and keyword index first. If the
|
||||
provider fails, analyze exits nonzero while those queries remain available. Run
|
||||
plain `gitnexus analyze` or `gitnexus embeddings sync` to resume a retained job.
|
||||
Compatible acknowledged work is reused without provider requests; an interrupted
|
||||
unacknowledged unit may replay. Changing resource settings does not invalidate
|
||||
paid vectors. `gitnexus status --json` exposes graph freshness and embedding
|
||||
progress separately; semantic search uses a completed, validated vector artifact.
|
||||
|
||||
Escape hatches (`GITNEXUS_MEMORY=off` to decline the autopilot,
|
||||
`GITNEXUS_WORKER_HEAP_MB` to size workers yourself) are listed in the
|
||||
environment-variable table below —
|
||||
|
|
@ -959,6 +976,7 @@ Four env vars expose the pool's resilience layers (respawn budget, cumulative-ti
|
|||
| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). |
|
||||
| `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. |
|
||||
| `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. |
|
||||
| `GITNEXUS_ANALYSIS_SUPERVISOR_BYTES` | nonnegative integer bytes | automatic | Internal inherited reservation for the waiting parent process. Leave unset for automatic accounting; a supplied value counts toward analysis admission and does not set the parent heap. |
|
||||
| `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. |
|
||||
| `GITNEXUS_HEAP_LIMIT_SOURCE` | unset | Set by analyze itself (`budget` or `auto`) to record where the heap limit came from, so out-of-memory advice points at `--memory-budget` rather than a `--max-old-space-size` pin. | Never — internal; set `--memory-budget` instead. |
|
||||
| `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. |
|
||||
|
|
|
|||
61
gitnexus/bench/analyze-resume/README.md
Normal file
61
gitnexus/bench/analyze-resume/README.md
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Resumable analyze benchmark
|
||||
|
||||
Build two checkouts with matching dependencies on the same filesystem, then run
|
||||
from the candidate's `gitnexus/` directory:
|
||||
|
||||
```sh
|
||||
node bench/analyze-resume/measure.mjs --check \
|
||||
--baseline /path/to/base/gitnexus/dist/cli/index.js \
|
||||
--output /tmp/analyze-resume-explicit.json
|
||||
```
|
||||
|
||||
Repeat with `--default-resources` and a new output path to compare automatic
|
||||
resource allocation. The explicit profile uses a 2 GiB heap, two parser workers,
|
||||
and a 256 MiB native pool on both sides. Do not run indexing or tests alongside
|
||||
these measurements.
|
||||
|
||||
`baseline.json` records the corpus settings, seven measured pairs per scenario,
|
||||
commit IDs, and original Linux/Node 22 results. Its historical timings are
|
||||
provenance, not portable wall-clock limits. The runner compares the supplied base
|
||||
and candidate in alternating order, after a warmup, on the current machine.
|
||||
|
||||
The six CI scenarios cover graph-only analysis, warm no-op, an incremental edit,
|
||||
fresh HTTP embeddings, cached HTTP embeddings, and retry after a deterministic
|
||||
provider outage. Setup runs are outside the measured interval. Assertions require
|
||||
equal graph/vector counts and embedding-input fingerprints, recorded runtime
|
||||
dependencies, completed publication, and no paid requests for a cached run.
|
||||
|
||||
`--check` requires all six scenarios, the recorded corpus size, a warmup and at
|
||||
least seven pairs. It fails when the paired 95% interval lies entirely
|
||||
above the candidate/base ratio of **1.0**. The interval encloses both the
|
||||
percentile bootstrap and Student-t interval on paired log ratios, avoiding the
|
||||
narrow bootstrap-only interval with seven samples. Student-t inference assumes
|
||||
independent pairs with approximately normal log ratios
|
||||
([NIST confidence interval definition](https://www.itl.nist.gov/div898/handbook/eda/section3/eda352.htm)). An interval crossing 1 remains
|
||||
**inconclusive** and is reported as such; a successful CI gate means no slowdown
|
||||
was statistically detected, not proof of equal performance. The original
|
||||
graph-only interval in `baseline.json` is inconclusive and has not been discarded.
|
||||
|
||||
`.github/workflows/ci-tests.yml` runs both resource profiles against the PR base
|
||||
(the first parent on non-PR runs). Failures block the existing CI Gate. Each job
|
||||
uses `--allow-dependency-changes` to measure the complete shipped runtimes, so
|
||||
dependency updates and release-version changes are benchmarked as well. Reports
|
||||
record both lockfile/native digests and each pair's resolved dependency receipts
|
||||
and parity. Manual runs require identical dependencies unless this flag is set.
|
||||
Each job runs `--validate-gate` decision checks before measuring and
|
||||
uploads raw samples, summaries, dependency receipts and analyzer logs, including
|
||||
partial results on failure. The runner never overwrites an existing report.
|
||||
|
||||
The explicit-profile CI job also uploads `analyze-resume-cpu-profile-explicit` before the
|
||||
full comparison. This separate one-pair fresh-HTTP run uses `--cpu-profile` to
|
||||
capture Node CPU profiles for each process and parser worker. Profiling is
|
||||
incompatible with `--check`; acceptance timings always run without profiling.
|
||||
|
||||
Benchmark execution belongs exclusively to the dedicated CI jobs above. It is
|
||||
not included in the unit or integration test suites.
|
||||
|
||||
`--help` lists additional options, including optional local-model scenarios.
|
||||
Synthetic fixtures do not establish query ranking, concurrent query latency,
|
||||
local-model throughput, exact peak RSS or platform correctness. Reports keep
|
||||
overall acceptance marked incomplete. SIGINT sent to the harness stops between
|
||||
invocations and lets any active native CLI exit naturally.
|
||||
256
gitnexus/bench/analyze-resume/baseline.json
Normal file
256
gitnexus/bench/analyze-resume/baseline.json
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"maxPairedRatio": 1,
|
||||
"settings": {
|
||||
"pairs": 7,
|
||||
"warmups": 1,
|
||||
"modules": 64,
|
||||
"functions": 16,
|
||||
"heap-mb": 2048,
|
||||
"workers": 2,
|
||||
"sample-ms": 25
|
||||
},
|
||||
"reference": {
|
||||
"measuredAt": "2026-11-05T19:17:51.910Z",
|
||||
"baselineCommit": "a0586d967f391f6f78e24d8c7162abfe350ae195",
|
||||
"candidateCommit": "8c1b489c962a223b77567c21dcd1a9f2c9788851",
|
||||
"environment": {
|
||||
"node": "v22.23.3",
|
||||
"platform": "linux",
|
||||
"architecture": "x64",
|
||||
"cpu": "12th Gen Intel(R) Core(TM) i9-12900K",
|
||||
"resourceProfile": "explicit-heap-workers-pool"
|
||||
},
|
||||
"note": "Historical samples are provenance, not portable wall-clock limits. CI compares the current PR base and candidate on the same runner. An interval crossing 1 remains inconclusive."
|
||||
},
|
||||
"scenarios": {
|
||||
"graph-only": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 11254.142376000003,
|
||||
"candidateMedianMs": 10191.042332999998,
|
||||
"geometricRatio": 1.0582904590478737,
|
||||
"pairedBootstrap95": [0.9484949043112423, 1.2159205271380682],
|
||||
"evidence": "inconclusive",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 11254.142376000003,
|
||||
"candidate": 10144.616939999996
|
||||
},
|
||||
{
|
||||
"baseline": 10671.354456000001,
|
||||
"candidate": 9944.314222
|
||||
},
|
||||
{
|
||||
"baseline": 10188.076402000006,
|
||||
"candidate": 10191.042332999998
|
||||
},
|
||||
{
|
||||
"baseline": 11853.487859999994,
|
||||
"candidate": 17919.62693900001
|
||||
},
|
||||
{
|
||||
"baseline": 11739.811645999987,
|
||||
"candidate": 14308.022194000005
|
||||
},
|
||||
{
|
||||
"baseline": 10668.565815000009,
|
||||
"candidate": 10129.661259000015
|
||||
},
|
||||
{
|
||||
"baseline": 11425.665953999996,
|
||||
"candidate": 11556.346957000002
|
||||
}
|
||||
]
|
||||
},
|
||||
"noop": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 2184.29468500003,
|
||||
"candidateMedianMs": 1950.1828749999986,
|
||||
"geometricRatio": 0.8930352942528001,
|
||||
"pairedBootstrap95": [0.8795970798771887, 0.905936077701467],
|
||||
"evidence": "no-detected-regression",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 2198.722995999997,
|
||||
"candidate": 1915.772708000004
|
||||
},
|
||||
{
|
||||
"baseline": 2239.4145290000015,
|
||||
"candidate": 1939.7219499999774
|
||||
},
|
||||
{
|
||||
"baseline": 2176.4498259999673,
|
||||
"candidate": 1950.1828749999986
|
||||
},
|
||||
{
|
||||
"baseline": 2184.29468500003,
|
||||
"candidate": 2005.5672990000458
|
||||
},
|
||||
{
|
||||
"baseline": 2179.4396279999637,
|
||||
"candidate": 1966.6888930000132
|
||||
},
|
||||
{
|
||||
"baseline": 2128.4910190000082,
|
||||
"candidate": 1891.617927000043
|
||||
},
|
||||
{
|
||||
"baseline": 2208.1911739999778,
|
||||
"candidate": 2008.7850609999732
|
||||
}
|
||||
]
|
||||
},
|
||||
"incremental": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 5783.258446000051,
|
||||
"candidateMedianMs": 5541.312566999986,
|
||||
"geometricRatio": 0.9674187960203702,
|
||||
"pairedBootstrap95": [0.9561405967756264, 0.9816590666894027],
|
||||
"evidence": "no-detected-regression",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 5779.961653999984,
|
||||
"candidate": 5672.744095000031
|
||||
},
|
||||
{
|
||||
"baseline": 5806.2170550000155,
|
||||
"candidate": 5541.312566999986
|
||||
},
|
||||
{
|
||||
"baseline": 5861.75522000005,
|
||||
"candidate": 5892.487050000054
|
||||
},
|
||||
{
|
||||
"baseline": 5720.598826000001,
|
||||
"candidate": 5503.345939000021
|
||||
},
|
||||
{
|
||||
"baseline": 5744.461226000101,
|
||||
"candidate": 5528.829626999912
|
||||
},
|
||||
{
|
||||
"baseline": 5783.258446000051,
|
||||
"candidate": 5528.050583000062
|
||||
},
|
||||
{
|
||||
"baseline": 5888.105557000032,
|
||||
"candidate": 5603.379379000049
|
||||
}
|
||||
]
|
||||
},
|
||||
"http-fresh": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 15511.024734000093,
|
||||
"candidateMedianMs": 14335.91775100003,
|
||||
"geometricRatio": 0.9272010938503702,
|
||||
"pairedBootstrap95": [0.918360962384628, 0.9365473751189902],
|
||||
"evidence": "no-detected-regression",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 15511.024734000093,
|
||||
"candidate": 14335.91775100003
|
||||
},
|
||||
{
|
||||
"baseline": 15510.167916000006,
|
||||
"candidate": 14261.329035999952
|
||||
},
|
||||
{
|
||||
"baseline": 15609.408870000043,
|
||||
"candidate": 14153.163587999996
|
||||
},
|
||||
{
|
||||
"baseline": 15409.513349999906,
|
||||
"candidate": 14634.943147000042
|
||||
},
|
||||
{
|
||||
"baseline": 15519.041560999933,
|
||||
"candidate": 14481.790279999957
|
||||
},
|
||||
{
|
||||
"baseline": 15767.326278000022,
|
||||
"candidate": 14690.776563999942
|
||||
},
|
||||
{
|
||||
"baseline": 15408.597242999938,
|
||||
"candidate": 14267.204865999985
|
||||
}
|
||||
]
|
||||
},
|
||||
"http-cached": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 2149.4189200000837,
|
||||
"candidateMedianMs": 1919.7719739999156,
|
||||
"geometricRatio": 0.8972216499596883,
|
||||
"pairedBootstrap95": [0.8876158802568739, 0.9067963765721975],
|
||||
"evidence": "no-detected-regression",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 2165.3657219999004,
|
||||
"candidate": 1909.4434619999956
|
||||
},
|
||||
{
|
||||
"baseline": 2132.151501000044,
|
||||
"candidate": 1941.053768999991
|
||||
},
|
||||
{
|
||||
"baseline": 2141.9542270000093,
|
||||
"candidate": 1934.437436999986
|
||||
},
|
||||
{
|
||||
"baseline": 2155.985989000066,
|
||||
"candidate": 1919.7719739999156
|
||||
},
|
||||
{
|
||||
"baseline": 2113.9462319999,
|
||||
"candidate": 1902.646522999974
|
||||
},
|
||||
{
|
||||
"baseline": 2149.4189200000837,
|
||||
"candidate": 1968.86528300005
|
||||
},
|
||||
{
|
||||
"baseline": 2152.111167999916,
|
||||
"candidate": 1892.6074149999768
|
||||
}
|
||||
]
|
||||
},
|
||||
"http-resume": {
|
||||
"pairs": 7,
|
||||
"baselineMedianMs": 15869.910479000071,
|
||||
"candidateMedianMs": 4004.7973079998046,
|
||||
"geometricRatio": 0.2485479851486286,
|
||||
"pairedBootstrap95": [0.24355429100174475, 0.25278581390854993],
|
||||
"evidence": "no-detected-regression",
|
||||
"samplesMs": [
|
||||
{
|
||||
"baseline": 15779.565613000188,
|
||||
"candidate": 4038.0054809998255
|
||||
},
|
||||
{
|
||||
"baseline": 15869.910479000071,
|
||||
"candidate": 4017.237259999849
|
||||
},
|
||||
{
|
||||
"baseline": 15861.843901999993,
|
||||
"candidate": 3971.7236019999254
|
||||
},
|
||||
{
|
||||
"baseline": 15861.11537200003,
|
||||
"candidate": 3973.9850310001057
|
||||
},
|
||||
{
|
||||
"baseline": 17113.716924000066,
|
||||
"candidate": 4044.0885320000816
|
||||
},
|
||||
{
|
||||
"baseline": 16508.58412100002,
|
||||
"candidate": 4004.7973079998046
|
||||
},
|
||||
{
|
||||
"baseline": 15901.872792999959,
|
||||
"candidate": 3999.7262199998368
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
824
gitnexus/bench/analyze-resume/measure.mjs
Normal file
824
gitnexus/bench/analyze-resume/measure.mjs
Normal file
|
|
@ -0,0 +1,824 @@
|
|||
/** Whole-command paired measurements. Run --help; never treats a smoke run as acceptance. */
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { createServer } from 'node:http';
|
||||
import { createRequire } from 'node:module';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { parseArgs } from 'node:util';
|
||||
|
||||
const SCENARIOS = [
|
||||
'graph-only',
|
||||
'noop',
|
||||
'incremental',
|
||||
'http-fresh',
|
||||
'http-cached',
|
||||
'http-resume',
|
||||
'local-fresh',
|
||||
'local-cached',
|
||||
];
|
||||
const sha = (text) => createHash('sha256').update(text).digest('hex');
|
||||
|
||||
async function runtimeEnvironment(cli) {
|
||||
const describeFile = async (file) => {
|
||||
const resolved = await fs.realpath(file);
|
||||
const [stat, filesystem] = await Promise.all([
|
||||
fs.stat(resolved, { bigint: true }),
|
||||
fs.statfs(resolved, { bigint: true }),
|
||||
]);
|
||||
return { path: resolved, device: String(stat.dev), filesystem: String(filesystem.type) };
|
||||
};
|
||||
const nativeEntry = createRequire(cli).resolve('@ladybugdb/core');
|
||||
return {
|
||||
cli: await describeFile(cli),
|
||||
nativeEntry: await describeFile(nativeEntry),
|
||||
nativeEntryDigest: sha(await fs.readFile(nativeEntry)),
|
||||
lockDigest: sha(await fs.readFile(path.resolve(path.dirname(cli), '../../package-lock.json'))),
|
||||
};
|
||||
}
|
||||
|
||||
export function assertComparableRuntimes(baseline, candidate, allowDependencyChanges = false) {
|
||||
for (const component of ['cli', 'nativeEntry']) {
|
||||
assert.equal(
|
||||
candidate[component].device,
|
||||
baseline[component].device,
|
||||
`${component} runtime files are on different devices; stage both checkouts on the same filesystem`,
|
||||
);
|
||||
assert.equal(
|
||||
candidate[component].filesystem,
|
||||
baseline[component].filesystem,
|
||||
`${component} runtime files are on different filesystem types`,
|
||||
);
|
||||
}
|
||||
if (!allowDependencyChanges) {
|
||||
assert.equal(candidate.lockDigest, baseline.lockDigest, 'Dependency lockfiles differ');
|
||||
assert.equal(
|
||||
candidate.nativeEntryDigest,
|
||||
baseline.nativeEntryDigest,
|
||||
'Native runtime entry bytes differ',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertCompletedCase(scenario, metadata, beforeMetadata) {
|
||||
assert(
|
||||
!metadata.incrementalInProgress && !metadata.graphWriteCollapsed,
|
||||
'Graph was not finalized',
|
||||
);
|
||||
assert(
|
||||
Number.isSafeInteger(metadata.stats?.nodes) && metadata.stats.nodes > 0,
|
||||
'Missing graph node count',
|
||||
);
|
||||
if (scenario === 'noop') {
|
||||
assert(beforeMetadata?.indexedAt, 'No-op setup lacks a freshness receipt');
|
||||
assert.equal(metadata.indexedAt, beforeMetadata.indexedAt, 'No-op changed graph freshness');
|
||||
}
|
||||
if (scenario.startsWith('http-') || scenario.startsWith('local-')) {
|
||||
assert(metadata.stats.embeddings > 0, 'Embedding scenario produced no vectors');
|
||||
assert(
|
||||
!metadata.embeddingCheckpoint && !metadata.embeddingJob,
|
||||
'Embedding work remains pending',
|
||||
);
|
||||
}
|
||||
}
|
||||
const median = (values) => {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
||||
};
|
||||
|
||||
/** Paired log-ratio intervals. Crossing 1 is unresolved, never equivalence. */
|
||||
export function summarizePairs(pairs) {
|
||||
assert(pairs.length > 0, 'At least one measured pair is required');
|
||||
const logs = pairs.map(({ baseline, candidate }) => {
|
||||
assert(
|
||||
Number.isFinite(baseline) && baseline > 0 && Number.isFinite(candidate) && candidate > 0,
|
||||
);
|
||||
return Math.log(candidate / baseline);
|
||||
});
|
||||
const average = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
|
||||
let seed = 3598;
|
||||
const random = () => {
|
||||
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
|
||||
return seed / 2 ** 32;
|
||||
};
|
||||
const samples = Array.from({ length: 10_000 }, () =>
|
||||
Math.exp(average(logs.map(() => logs[Math.floor(random() * logs.length)]))),
|
||||
).sort((a, b) => a - b);
|
||||
const bootstrap = [samples[250], samples[9749]];
|
||||
// Bootstrap alone understates uncertainty with seven pairs. Enclose it in
|
||||
// the Student-t interval for the mean log ratio; round df down in this table.
|
||||
const df = logs.length - 1;
|
||||
const critical = [
|
||||
[1, 12.707],
|
||||
[2, 4.303],
|
||||
[3, 3.183],
|
||||
[4, 2.777],
|
||||
[5, 2.571],
|
||||
[6, 2.447],
|
||||
[7, 2.365],
|
||||
[8, 2.307],
|
||||
[9, 2.263],
|
||||
[10, 2.229],
|
||||
[15, 2.132],
|
||||
[20, 2.086],
|
||||
[30, 2.043],
|
||||
]
|
||||
.filter(([degree]) => degree <= df)
|
||||
.at(-1)?.[1];
|
||||
const mean = average(logs);
|
||||
const margin =
|
||||
df > 0
|
||||
? critical * Math.sqrt(logs.reduce((sum, x) => sum + (x - mean) ** 2, 0) / df / logs.length)
|
||||
: 0;
|
||||
const student = [Math.exp(mean - margin), Math.exp(mean + margin)];
|
||||
const interval = [Math.min(bootstrap[0], student[0]), Math.max(bootstrap[1], student[1])];
|
||||
return {
|
||||
pairs: pairs.length,
|
||||
baselineMedianMs: median(pairs.map((p) => p.baseline)),
|
||||
candidateMedianMs: median(pairs.map((p) => p.candidate)),
|
||||
geometricRatio: Math.exp(average(logs)),
|
||||
pairedBootstrap95: bootstrap,
|
||||
pairedStudent95: student,
|
||||
paired95: interval,
|
||||
evidence:
|
||||
pairs.length < 7
|
||||
? 'smoke-only'
|
||||
: interval[0] > 1
|
||||
? 'regression'
|
||||
: interval[1] < 1
|
||||
? 'no-detected-regression'
|
||||
: 'inconclusive',
|
||||
};
|
||||
}
|
||||
|
||||
/** Detect slowdowns against a contemporaneous base, never historical wall-clock timings. */
|
||||
export function checkPerformance(summaries, baseline) {
|
||||
assert.equal(baseline.schemaVersion, 1, 'Unsupported benchmark baseline');
|
||||
assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression');
|
||||
const scenarios = Object.keys(baseline.scenarios);
|
||||
assert(scenarios.length > 0, 'Missing baseline scenarios');
|
||||
const failedScenarios = [];
|
||||
const inconclusiveScenarios = [];
|
||||
for (const scenario of scenarios) {
|
||||
const summary = summaries[scenario];
|
||||
assert(summary && summary.pairs >= 7, `Missing seven measured pairs for ${scenario}`);
|
||||
const interval = summary.paired95;
|
||||
assert(
|
||||
Array.isArray(interval) &&
|
||||
interval.length === 2 &&
|
||||
interval.every((value) => Number.isFinite(value) && value > 0) &&
|
||||
interval[0] <= interval[1],
|
||||
`Invalid paired interval for ${scenario}`,
|
||||
);
|
||||
if (interval[0] > baseline.maxPairedRatio) failedScenarios.push(scenario);
|
||||
else if (interval[1] >= baseline.maxPairedRatio) inconclusiveScenarios.push(scenario);
|
||||
}
|
||||
return { failedScenarios, inconclusiveScenarios };
|
||||
}
|
||||
|
||||
/** Sample Linux process RSS once per process, including descendants, never per thread. */
|
||||
async function sampleTree(pid) {
|
||||
if (process.platform !== 'linux') return undefined;
|
||||
const pending = [pid];
|
||||
const seen = new Set();
|
||||
let bytes = 0;
|
||||
while (pending.length) {
|
||||
const current = pending.pop();
|
||||
if (seen.has(current)) continue;
|
||||
seen.add(current);
|
||||
try {
|
||||
const [status, children] = await Promise.all([
|
||||
fs.readFile(`/proc/${current}/status`, 'utf8'),
|
||||
fs.readFile(`/proc/${current}/task/${current}/children`, 'utf8'),
|
||||
]);
|
||||
bytes += Number(status.match(/^VmRSS:\s+(\d+)\s+kB$/m)?.[1] ?? 0) * 1024;
|
||||
pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number));
|
||||
} catch (error) {
|
||||
if (!['ENOENT', 'ESRCH'].includes(error.code)) throw error;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export async function measureProcess(command, args, { cwd, env, logFile, sampleMs = 25 }) {
|
||||
const log = await fs.open(logFile, 'wx');
|
||||
let timer;
|
||||
let polling;
|
||||
let samplingError;
|
||||
let peakRssBytes = process.platform === 'linux' ? 0 : null;
|
||||
let samples = 0;
|
||||
const start = performance.now();
|
||||
try {
|
||||
const child = spawn(command, args, { cwd, env, stdio: ['ignore', log.fd, log.fd] });
|
||||
const poll = () => {
|
||||
if (!polling && child.pid) {
|
||||
polling = sampleTree(child.pid)
|
||||
.then((rss) => {
|
||||
if (rss !== undefined) {
|
||||
peakRssBytes = Math.max(peakRssBytes, rss);
|
||||
samples++;
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
samplingError ??= error.message;
|
||||
})
|
||||
.finally(() => {
|
||||
polling = undefined;
|
||||
});
|
||||
}
|
||||
};
|
||||
poll();
|
||||
timer = setInterval(poll, sampleMs);
|
||||
const outcome = await new Promise((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('close', (exitCode, signal) => resolve({ exitCode, signal }));
|
||||
});
|
||||
const elapsedMs = performance.now() - start;
|
||||
clearInterval(timer);
|
||||
await polling;
|
||||
return {
|
||||
...outcome,
|
||||
elapsedMs,
|
||||
sampledPeakProcessTreeRssBytes: samplingError ? null : peakRssBytes,
|
||||
rssSamples: samples,
|
||||
rssSampleIntervalMs: sampleMs,
|
||||
samplingError,
|
||||
logFile,
|
||||
};
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
await polling;
|
||||
await log.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function startProvider(dimensions = 256) {
|
||||
let active;
|
||||
const server = createServer(async (req, res) => {
|
||||
const counters = active;
|
||||
try {
|
||||
assert(counters && req.method === 'POST', 'Unexpected provider request');
|
||||
let size = 0;
|
||||
const chunks = [];
|
||||
for await (const chunk of req) {
|
||||
size += chunk.length;
|
||||
assert(size <= 8 * 1024 * 1024, 'Benchmark request exceeds 8 MiB');
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const { input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||
assert(
|
||||
Array.isArray(input) && input.length && input.every((text) => typeof text === 'string'),
|
||||
);
|
||||
counters.requests++;
|
||||
counters.requestBytes += size;
|
||||
if (counters.acceptedInputs + input.length > counters.failAfter) {
|
||||
counters.rejectedRequests++;
|
||||
res.writeHead(503, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: { message: 'Deterministic interrupted-run fixture' } }));
|
||||
return;
|
||||
}
|
||||
const data = input.map((text, index) => {
|
||||
const hash = createHash('sha256').update(text).digest();
|
||||
counters.inputs.add(hash.toString('hex'));
|
||||
counters.acceptedInputs++;
|
||||
// Same text produces exactly the same nonzero vector in either implementation.
|
||||
const embedding = Array.from({ length: dimensions }, (_, i) => (hash[i % 32] + 1) / 256);
|
||||
return { object: 'embedding', index, embedding };
|
||||
});
|
||||
const body = JSON.stringify({ object: 'list', data, model: 'gitnexus-benchmark' });
|
||||
counters.responseBytes += Buffer.byteLength(body);
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(body);
|
||||
} catch (error) {
|
||||
if (counters) counters.error ??= error.message;
|
||||
res.writeHead(400, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: { message: error.message } }));
|
||||
}
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', resolve);
|
||||
});
|
||||
return {
|
||||
url: `http://127.0.0.1:${server.address().port}/v1`,
|
||||
begin(failAfter = Infinity) {
|
||||
active = {
|
||||
failAfter,
|
||||
requests: 0,
|
||||
rejectedRequests: 0,
|
||||
acceptedInputs: 0,
|
||||
requestBytes: 0,
|
||||
responseBytes: 0,
|
||||
inputs: new Set(),
|
||||
};
|
||||
return active;
|
||||
},
|
||||
close: () =>
|
||||
new Promise((resolve, reject) =>
|
||||
server.close((error) => (error ? reject(error) : resolve())),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function moduleText(moduleIndex, modules, functions, revision = 0) {
|
||||
const previous = (moduleIndex + modules - 1) % modules;
|
||||
const lines = [`import { task_${previous}_0 } from './module-${previous}.js';`];
|
||||
for (let i = 0; i < functions; i++) {
|
||||
const dependency = i ? `task_${moduleIndex}_${i - 1}` : `task_${previous}_0`;
|
||||
lines.push(`/** Compute deterministic module ${moduleIndex}, operation ${i}. */`);
|
||||
lines.push(`export function task_${moduleIndex}_${i}(value: number): number {`);
|
||||
lines.push(` if (value <= 0) return ${i + (moduleIndex === 0 && i === 0 ? revision : 0)};`);
|
||||
lines.push(` return ${dependency}(value - 1) + ${i + 1};`, '}');
|
||||
}
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
async function fixture(directory, modules, functions) {
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(path.join(directory, 'package.json'), '{"type":"module"}\n');
|
||||
for (let i = 0; i < modules; i++) {
|
||||
await fs.writeFile(path.join(directory, `module-${i}.ts`), moduleText(i, modules, functions));
|
||||
}
|
||||
const git = (...args) =>
|
||||
execFileSync('git', args, {
|
||||
cwd: directory,
|
||||
stdio: 'pipe',
|
||||
env: {
|
||||
...process.env,
|
||||
GIT_AUTHOR_DATE: '2026-01-01T00:00:00Z',
|
||||
GIT_COMMITTER_DATE: '2026-01-01T00:00:00Z',
|
||||
},
|
||||
});
|
||||
git('init', '-q');
|
||||
git('config', 'user.name', 'GitNexus benchmark');
|
||||
git('config', 'user.email', 'benchmark@example.invalid');
|
||||
git('config', 'commit.gpgsign', 'false');
|
||||
const hooks = path.join(directory, '.git', 'empty-hooks');
|
||||
await fs.mkdir(hooks);
|
||||
git('config', 'core.hooksPath', hooks);
|
||||
git('add', '.');
|
||||
git('commit', '-qm', 'fixture');
|
||||
return git;
|
||||
}
|
||||
|
||||
function providerSnapshot(counters) {
|
||||
return {
|
||||
requests: counters.requests,
|
||||
rejectedRequests: counters.rejectedRequests,
|
||||
acceptedInputs: counters.acceptedInputs,
|
||||
uniqueInputs: counters.inputs.size,
|
||||
uniqueInputFingerprint: sha([...counters.inputs].sort().join('\n')),
|
||||
requestBytes: counters.requestBytes,
|
||||
responseBytes: counters.responseBytes,
|
||||
error: counters.error,
|
||||
};
|
||||
}
|
||||
|
||||
async function runCase({
|
||||
cli,
|
||||
scenario,
|
||||
directory,
|
||||
modules,
|
||||
functions,
|
||||
heapMb,
|
||||
workers,
|
||||
defaultResources,
|
||||
cpuProfile,
|
||||
provider,
|
||||
sampleMs,
|
||||
stopRequested,
|
||||
}) {
|
||||
const repo = path.join(directory, 'repo');
|
||||
const store = path.join(directory, 'store');
|
||||
const git = await fixture(repo, modules, functions);
|
||||
const env = { ...process.env };
|
||||
// Isolate GitNexus configuration/registry/credentials; never read a user's real config.
|
||||
for (const key of Object.keys(env)) if (key.startsWith('GITNEXUS_')) delete env[key];
|
||||
delete env.NODE_OPTIONS;
|
||||
Object.assign(env, {
|
||||
GITNEXUS_HOME: path.join(directory, 'home'),
|
||||
GITNEXUS_STORAGE_PATH: store,
|
||||
GITNEXUS_SHARED_STORE: 'off',
|
||||
GITNEXUS_WORKER_READY_TIMEOUT_MS: '30000',
|
||||
});
|
||||
if (!defaultResources) env.GITNEXUS_LBUG_BUFFER_POOL_SIZE = String(256 * 1024 * 1024);
|
||||
const http = scenario.startsWith('http-');
|
||||
const embedding = http || scenario.startsWith('local-');
|
||||
if (http)
|
||||
Object.assign(env, {
|
||||
GITNEXUS_EMBEDDING_URL: provider.url,
|
||||
GITNEXUS_EMBEDDING_MODEL: 'gitnexus-benchmark',
|
||||
GITNEXUS_EMBEDDING_DIMS: '256',
|
||||
GITNEXUS_EMBEDDING_API_KEY: 'benchmark-only',
|
||||
GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1',
|
||||
GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '16',
|
||||
});
|
||||
const args = [
|
||||
...(cpuProfile ? ['--cpu-prof', `--cpu-prof-dir=${directory}`] : []),
|
||||
cli,
|
||||
'analyze',
|
||||
repo,
|
||||
'--index-only',
|
||||
];
|
||||
if (!defaultResources) args.push('--memory-budget', String(heapMb), '--workers', String(workers));
|
||||
if (embedding) args.push('--embeddings', '0');
|
||||
const counters = provider.begin(
|
||||
scenario === 'http-resume' ? Math.max(16, Math.floor((modules * functions) / 3)) : Infinity,
|
||||
);
|
||||
const invoke = async (name, allowFailure = false) => {
|
||||
assert(
|
||||
!stopRequested(),
|
||||
'Benchmark interrupted between invocations; native children were not signalled',
|
||||
);
|
||||
const result = await measureProcess(process.execPath, args, {
|
||||
cwd: repo,
|
||||
env,
|
||||
logFile: path.join(directory, `${name}.log`),
|
||||
sampleMs,
|
||||
});
|
||||
if (!allowFailure)
|
||||
assert.equal(result.exitCode, 0, `Analyze failed; inspect ${result.logFile}`);
|
||||
assert(!result.signal, `Analyze ended with ${result.signal}; inspect ${result.logFile}`);
|
||||
assert(!counters.error, counters.error);
|
||||
return result;
|
||||
};
|
||||
let setup;
|
||||
if (['noop', 'incremental', 'http-cached', 'http-resume', 'local-cached'].includes(scenario)) {
|
||||
setup = await invoke('setup', scenario === 'http-resume');
|
||||
if (scenario === 'http-resume') {
|
||||
assert(
|
||||
counters.rejectedRequests > 0 && counters.acceptedInputs > 0,
|
||||
'Resume setup must actually interrupt partially successful work',
|
||||
);
|
||||
counters.failAfter = Infinity;
|
||||
}
|
||||
if (scenario === 'incremental') {
|
||||
await fs.writeFile(path.join(repo, 'module-0.ts'), moduleText(0, modules, functions, 1));
|
||||
git('add', 'module-0.ts');
|
||||
git('commit', '-qm', 'one-function edit');
|
||||
}
|
||||
}
|
||||
const beforeMetadata =
|
||||
setup && scenario !== 'http-resume'
|
||||
? JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8'))
|
||||
: undefined;
|
||||
const before = providerSnapshot(counters);
|
||||
const measurement = await invoke('measured');
|
||||
const after = providerSnapshot(counters);
|
||||
if (scenario === 'http-cached')
|
||||
assert.equal(after.acceptedInputs, before.acceptedInputs, 'Cached run made paid requests');
|
||||
const metadata = JSON.parse(await fs.readFile(path.join(store, 'gitnexus.json'), 'utf8'));
|
||||
assertCompletedCase(scenario, metadata, beforeMetadata);
|
||||
const stats = metadata.stats;
|
||||
const result = {
|
||||
...measurement,
|
||||
setup,
|
||||
command: [process.execPath, ...args],
|
||||
providerBefore: before,
|
||||
providerAfter: after,
|
||||
measuredProviderRequests: after.requests - before.requests,
|
||||
measuredProviderInputs: after.acceptedInputs - before.acceptedInputs,
|
||||
equalWork: {
|
||||
files: stats.files,
|
||||
nodes: stats.nodes,
|
||||
edges: stats.edges,
|
||||
embeddings: stats.embeddings ?? 0,
|
||||
uniqueInputs: after.uniqueInputs,
|
||||
inputFingerprint: after.uniqueInputFingerprint,
|
||||
},
|
||||
metadata: {
|
||||
runnerIdentity: metadata.runnerIdentity,
|
||||
graphGeneration: metadata.graphGeneration,
|
||||
vectorGeneration: metadata.vectorGeneration,
|
||||
embeddingCheckpoint: metadata.embeddingCheckpoint,
|
||||
embeddingJob: metadata.embeddingJob,
|
||||
},
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(directory, 'measurement.json'),
|
||||
JSON.stringify(result, null, 2) + '\n',
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const baseline = JSON.parse(
|
||||
await fs.readFile(new URL('./baseline.json', import.meta.url), 'utf8'),
|
||||
);
|
||||
const { values: v } = parseArgs({
|
||||
args: argv,
|
||||
options: {
|
||||
baseline: { type: 'string' },
|
||||
candidate: { type: 'string' },
|
||||
output: { type: 'string' },
|
||||
scenarios: { type: 'string', default: Object.keys(baseline.scenarios).join(',') },
|
||||
pairs: { type: 'string', default: '7' },
|
||||
warmups: { type: 'string', default: '1' },
|
||||
modules: { type: 'string', default: '64' },
|
||||
functions: { type: 'string', default: '16' },
|
||||
'heap-mb': { type: 'string', default: '2048' },
|
||||
workers: { type: 'string', default: '2' },
|
||||
'default-resources': { type: 'boolean', default: false },
|
||||
'cpu-profile': { type: 'boolean', default: false },
|
||||
'allow-dependency-changes': { type: 'boolean', default: false },
|
||||
'sample-ms': { type: 'string', default: '25' },
|
||||
'keep-stores': { type: 'boolean', default: false },
|
||||
check: { type: 'boolean', default: false },
|
||||
'validate-gate': { type: 'boolean' },
|
||||
help: { type: 'boolean' },
|
||||
},
|
||||
});
|
||||
assert(
|
||||
!(v.check && v['cpu-profile']),
|
||||
'CPU profiles are diagnostic runs, not acceptance timings',
|
||||
);
|
||||
if (v['validate-gate']) {
|
||||
const fixture = { schemaVersion: 1, maxPairedRatio: 1, scenarios: { case: {} } };
|
||||
const summary = (ratios) =>
|
||||
summarizePairs(ratios.map((candidate) => ({ baseline: 1, candidate })));
|
||||
assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(1.1)) }, fixture), {
|
||||
failedScenarios: ['case'],
|
||||
inconclusiveScenarios: [],
|
||||
});
|
||||
assert.deepEqual(checkPerformance({ case: summary(Array(7).fill(0.9)) }, fixture), {
|
||||
failedScenarios: [],
|
||||
inconclusiveScenarios: [],
|
||||
});
|
||||
const mixed = summary([0.8, 0.9, 0.95, 1, 1.05, 1.1, 1.2]);
|
||||
assert.deepEqual(checkPerformance({ case: mixed }, fixture), {
|
||||
failedScenarios: [],
|
||||
inconclusiveScenarios: ['case'],
|
||||
});
|
||||
assert(mixed.paired95[0] <= mixed.pairedBootstrap95[0]);
|
||||
assert(mixed.paired95[1] >= mixed.pairedBootstrap95[1]);
|
||||
assert.throws(() => checkPerformance({ case: summary([1, 1]) }, fixture));
|
||||
assert.throws(() => summarizePairs([{ baseline: 0, candidate: 1 }]));
|
||||
const runtime = {
|
||||
cli: { device: '1', filesystem: '1' },
|
||||
nativeEntry: { device: '1', filesystem: '1' },
|
||||
lockDigest: 'a',
|
||||
nativeEntryDigest: 'a',
|
||||
};
|
||||
const changed = { ...runtime, lockDigest: 'b', nativeEntryDigest: 'b' };
|
||||
assert.throws(() => assertComparableRuntimes(runtime, changed));
|
||||
assertComparableRuntimes(runtime, changed, true);
|
||||
assert.throws(() =>
|
||||
assertComparableRuntimes(
|
||||
runtime,
|
||||
{ ...changed, cli: { device: '2', filesystem: '1' } },
|
||||
true,
|
||||
),
|
||||
);
|
||||
console.log('Paired gate decision checks passed');
|
||||
return;
|
||||
}
|
||||
if (v.help) {
|
||||
console.log(`Usage: node bench/analyze-resume/measure.mjs --baseline /repo/gitnexus/dist/cli/index.js --output /tmp/results.json
|
||||
--candidate PATH Defaults to this checkout's built CLI
|
||||
--scenarios LIST ${SCENARIOS.join(',')}
|
||||
--pairs 7 --warmups 1 Alternating paired invocations; fewer than seven pairs is smoke-only
|
||||
--modules 64 --functions 16 --heap-mb 2048 --workers 2 --sample-ms 25
|
||||
--default-resources Omit heap, worker and native-pool overrides on both sides
|
||||
--cpu-profile Retain per-process/thread CPU profiles; incompatible with --check
|
||||
--allow-dependency-changes Compare shipped runtimes, including intentional dependency changes
|
||||
--validate-gate Check decision logic without running benchmarks
|
||||
--check Require the baseline corpus/scenarios and reject detected slowdowns
|
||||
--keep-stores Retain owned index/home artifacts (logs and measurements always retained)
|
||||
Includes full CLI startup, graph/manifest/native writes and final index work.
|
||||
HTTP resume uses a deterministic provider outage, not an unsafe native-process kill.
|
||||
RSS is sampled Linux process-family RSS; other platforms report it unavailable.
|
||||
Local scenarios use the installed default model; prewarm its download before timing.
|
||||
Live-query latency and platform cases remain separate acceptance requirements.`);
|
||||
return;
|
||||
}
|
||||
assert(v.baseline && v.output, '--baseline and --output are required');
|
||||
const numbers = {};
|
||||
for (const key of [
|
||||
'pairs',
|
||||
'warmups',
|
||||
'modules',
|
||||
'functions',
|
||||
'heap-mb',
|
||||
'workers',
|
||||
'sample-ms',
|
||||
]) {
|
||||
numbers[key] = Number(v[key]);
|
||||
assert(
|
||||
Number.isSafeInteger(numbers[key]) && numbers[key] >= (key === 'warmups' ? 0 : 1),
|
||||
`Invalid --${key}`,
|
||||
);
|
||||
}
|
||||
assert(numbers.modules >= 2, '--modules must be at least 2');
|
||||
const scenarios = v.scenarios.split(',');
|
||||
assert(
|
||||
scenarios.length &&
|
||||
new Set(scenarios).size === scenarios.length &&
|
||||
scenarios.every((x) => SCENARIOS.includes(x)),
|
||||
'Invalid or repeated scenario',
|
||||
);
|
||||
if (v.check) {
|
||||
assert(numbers.pairs >= 7 && numbers.warmups >= 1, '--check requires seven pairs and a warmup');
|
||||
for (const key of ['modules', 'functions'])
|
||||
assert.equal(numbers[key], baseline.settings[key], `--check requires baseline ${key}`);
|
||||
assert(
|
||||
Object.keys(baseline.scenarios).every((scenario) => scenarios.includes(scenario)),
|
||||
'--check requires all baseline scenarios',
|
||||
);
|
||||
assert.equal(baseline.maxPairedRatio, 1, 'The gate must not allow a performance regression');
|
||||
}
|
||||
const candidate =
|
||||
v.candidate ?? fileURLToPath(new URL('../../dist/cli/index.js', import.meta.url));
|
||||
const clis = { baseline: path.resolve(v.baseline), candidate: path.resolve(candidate) };
|
||||
for (const cli of Object.values(clis)) await fs.access(cli);
|
||||
const runtime = {
|
||||
baseline: await runtimeEnvironment(clis.baseline),
|
||||
candidate: await runtimeEnvironment(clis.candidate),
|
||||
};
|
||||
assertComparableRuntimes(runtime.baseline, runtime.candidate, v['allow-dependency-changes']);
|
||||
const output = path.resolve(v.output);
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
// Never overwrite a prior result; evidence files and temporary directories are exclusively owned.
|
||||
const resultFile = await fs.open(output, 'wx');
|
||||
const work = await fs.mkdtemp(path.join(path.dirname(output), 'analyze-pairs-'));
|
||||
const provider = await startProvider();
|
||||
let interrupted = false;
|
||||
const interrupt = () => {
|
||||
interrupted = true;
|
||||
console.error(
|
||||
'Stopping between invocations; waiting for the active native CLI to exit normally.',
|
||||
);
|
||||
};
|
||||
process.on('SIGINT', interrupt);
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
createdAt: new Date().toISOString(),
|
||||
work,
|
||||
environment: {
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
cpu: os.cpus()[0]?.model,
|
||||
availableParallelism: os.availableParallelism(),
|
||||
totalMemory: os.totalmem(),
|
||||
settings: numbers,
|
||||
resourceProfile: v['default-resources'] ? 'default' : 'explicit-heap-workers-pool',
|
||||
cpuProfile: v['cpu-profile'],
|
||||
comparison: v['allow-dependency-changes'] ? 'shipped-runtimes' : 'identical-dependencies',
|
||||
runtime,
|
||||
clis: {},
|
||||
},
|
||||
limitations: [
|
||||
'Synthetic corpus only; stats and deterministic input coverage are checked, not query ranking.',
|
||||
'Linux RSS sampling is a lower-bound observation, not a hard memory limit or exact peak.',
|
||||
'Disk, phase timings, native reservations and queue peaks are not collected; optional CPU profiles are diagnostic only.',
|
||||
'Only the selected scenarios and resource profile were measured; live query latency and Windows require additional validation.',
|
||||
],
|
||||
acceptance: 'incomplete',
|
||||
samples: [],
|
||||
summaries: {},
|
||||
};
|
||||
const save = async () => {
|
||||
const bytes = Buffer.from(JSON.stringify(report, null, 2) + '\n');
|
||||
await resultFile.write(bytes, 0, bytes.length, 0);
|
||||
await resultFile.truncate(bytes.length);
|
||||
await resultFile.sync();
|
||||
};
|
||||
try {
|
||||
for (const [side, cli] of Object.entries(clis)) {
|
||||
report.environment.clis[side] = {
|
||||
path: cli,
|
||||
entryDigest: sha(await fs.readFile(cli)),
|
||||
commit: execFileSync('git', ['rev-parse', 'HEAD'], {
|
||||
cwd: path.dirname(cli),
|
||||
encoding: 'utf8',
|
||||
}).trim(),
|
||||
};
|
||||
}
|
||||
await save();
|
||||
for (const scenario of scenarios) {
|
||||
const pairs = [];
|
||||
for (let iteration = -numbers.warmups; iteration < numbers.pairs; iteration++) {
|
||||
const ordinal = iteration + numbers.warmups;
|
||||
const order = ordinal % 2 === 0 ? ['baseline', 'candidate'] : ['candidate', 'baseline'];
|
||||
const pair = { scenario, iteration, warmup: iteration < 0, order };
|
||||
report.samples.push(pair);
|
||||
for (const side of order) {
|
||||
assert(
|
||||
!interrupted,
|
||||
'Benchmark interrupted between invocations; partial evidence retained',
|
||||
);
|
||||
const directory = path.join(work, `${scenario}-${ordinal}-${side}`);
|
||||
pair[side] = await runCase({
|
||||
cli: clis[side],
|
||||
scenario,
|
||||
directory,
|
||||
modules: numbers.modules,
|
||||
functions: numbers.functions,
|
||||
heapMb: numbers['heap-mb'],
|
||||
workers: numbers.workers,
|
||||
defaultResources: v['default-resources'],
|
||||
cpuProfile: v['cpu-profile'],
|
||||
sampleMs: numbers['sample-ms'],
|
||||
provider,
|
||||
stopRequested: () => interrupted,
|
||||
});
|
||||
await save();
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
scenario,
|
||||
iteration,
|
||||
side,
|
||||
elapsedMs: pair[side].elapsedMs,
|
||||
requests: pair[side].measuredProviderRequests,
|
||||
rss: pair[side].sampledPeakProcessTreeRssBytes,
|
||||
}),
|
||||
);
|
||||
if (!v['keep-stores']) {
|
||||
// Only paths created inside this invocation's mkdtemp root are removed.
|
||||
await fs.rm(path.join(directory, 'store'), { recursive: true, force: true });
|
||||
await fs.rm(path.join(directory, 'home'), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
assert.deepEqual(
|
||||
pair.candidate.equalWork,
|
||||
pair.baseline.equalWork,
|
||||
`Unequal work for ${scenario}; inspect retained measurements`,
|
||||
);
|
||||
if (scenario === 'http-resume') {
|
||||
assert.equal(
|
||||
pair.candidate.providerAfter.acceptedInputs,
|
||||
pair.candidate.providerAfter.uniqueInputs,
|
||||
'Resume replayed already accepted embedding inputs',
|
||||
);
|
||||
assert.equal(
|
||||
pair.candidate.measuredProviderInputs,
|
||||
pair.candidate.providerAfter.uniqueInputs -
|
||||
pair.candidate.providerBefore.acceptedInputs,
|
||||
'Resume must request only unfinished embedding inputs',
|
||||
);
|
||||
}
|
||||
const baselineDependencies =
|
||||
pair.baseline.metadata.runnerIdentity?.dependencyRuntime?.digest;
|
||||
assert(
|
||||
/^sha256:[a-f0-9]{64}$/.test(baselineDependencies ?? ''),
|
||||
'Baseline dependency receipt missing',
|
||||
);
|
||||
const candidateDependencies =
|
||||
pair.candidate.metadata.runnerIdentity?.dependencyRuntime?.digest;
|
||||
assert(
|
||||
/^sha256:[a-f0-9]{64}$/.test(candidateDependencies ?? ''),
|
||||
'Candidate dependency receipt missing',
|
||||
);
|
||||
pair.dependenciesMatch = candidateDependencies === baselineDependencies;
|
||||
if (!v['allow-dependency-changes'])
|
||||
assert(
|
||||
pair.dependenciesMatch,
|
||||
'Resolved dependency payloads differ; use --allow-dependency-changes to compare shipped runtimes',
|
||||
);
|
||||
if (iteration >= 0)
|
||||
pairs.push({ baseline: pair.baseline.elapsedMs, candidate: pair.candidate.elapsedMs });
|
||||
}
|
||||
report.summaries[scenario] = summarizePairs(pairs);
|
||||
await save();
|
||||
}
|
||||
if (v.check) {
|
||||
report.check = checkPerformance(report.summaries, baseline);
|
||||
await save();
|
||||
assert.equal(
|
||||
report.check.failedScenarios.length,
|
||||
0,
|
||||
`Performance regression: ${report.check.failedScenarios.join(', ')}; inspect ${output}`,
|
||||
);
|
||||
if (report.check.inconclusiveScenarios.length)
|
||||
console.warn(
|
||||
`Inconclusive timings: ${report.check.inconclusiveScenarios.join(', ')}; not proof of equal performance.`,
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{ output, summaries: report.summaries, check: report.check, acceptance: report.acceptance },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
report.error = error.message;
|
||||
await save();
|
||||
throw error;
|
||||
} finally {
|
||||
process.removeListener('SIGINT', interrupt);
|
||||
await provider.close();
|
||||
await resultFile.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
|
@ -304,10 +304,21 @@ function isDirectChild(parent, child) {
|
|||
}
|
||||
|
||||
// Mirror gitnexus/src/storage/shared-store.ts resolveGraphPath (#3352): a
|
||||
// shared-store checkout slot may read a commit graph in the same store
|
||||
// instead of owning <slot>/lbug. Any other recorded value is ignored.
|
||||
// slot may read an owned private generation; shared checkout slots may also
|
||||
// read a commit graph in the same store. Other recorded values are ignored.
|
||||
function resolveGraphPath(storagePath, metadata) {
|
||||
const own = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded === 'string' && /^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) {
|
||||
const graph = path.join(storagePath, recorded);
|
||||
try {
|
||||
const stat = fs.lstatSync(graph);
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) return null;
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') return null;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
const storesRoot = path.resolve(
|
||||
process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'),
|
||||
'stores',
|
||||
|
|
@ -317,7 +328,6 @@ function resolveGraphPath(storagePath, metadata) {
|
|||
const root = path.dirname(checkoutsDir);
|
||||
if (path.basename(checkoutsDir) !== 'checkouts') return own;
|
||||
if (!isDirectChild(checkoutsDir, slot) || !isDirectChild(storesRoot, root)) return own;
|
||||
const recorded = metadata && metadata.graphPath;
|
||||
if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own;
|
||||
const graph = path.resolve(recorded);
|
||||
// Only a published `<commit>-<featureKey>` dir, never `.publish-*` staging.
|
||||
|
|
@ -353,11 +363,14 @@ function findLocalOwnedRepo(cwd) {
|
|||
const branchDir =
|
||||
branch != null ? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch)) : null;
|
||||
const indexDir = branchDir && hasLocalIndexSignal(branchDir) ? branchDir : storagePath;
|
||||
const indexMetadata = indexDir === storagePath ? metadata : readIndexMetadata(indexDir);
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) return null;
|
||||
return {
|
||||
path: current,
|
||||
storagePath,
|
||||
lbugPath: path.join(indexDir, LBUG_DIRECTORY),
|
||||
metadata: indexDir === storagePath ? metadata : readIndexMetadata(indexDir),
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -412,15 +425,16 @@ function findRegisteredRepo(cwd) {
|
|||
const indexDir = branchIsIndexed
|
||||
? path.join(storagePath, BRANCHES_DIRECTORY, branchSlug(branch))
|
||||
: storagePath;
|
||||
const indexMetadata = branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata;
|
||||
const lbugPath = resolveGraphPath(indexDir, indexMetadata);
|
||||
if (!lbugPath) continue;
|
||||
if (registeredPath.length > bestLen) {
|
||||
bestLen = registeredPath.length;
|
||||
best = {
|
||||
path: entry.path,
|
||||
storagePath,
|
||||
lbugPath: branchIsIndexed
|
||||
? path.join(indexDir, LBUG_DIRECTORY)
|
||||
: resolveGraphPath(storagePath, ownershipMetadata),
|
||||
metadata: branchIsIndexed ? readIndexMetadata(indexDir) : ownershipMetadata,
|
||||
lbugPath,
|
||||
metadata: indexMetadata,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,15 @@
|
|||
// Platform-specific logic tests — contain explicit process.platform guards
|
||||
// or test behavior that differs across operating systems
|
||||
const PLATFORM_LOGIC = [
|
||||
'test/unit/embedding-sidecar-lifetime.test.ts',
|
||||
'test/unit/analysis-publication.test.ts',
|
||||
'test/unit/analysis-phase-supervisor.test.ts',
|
||||
'test/unit/shared-vector-artifact.test.ts',
|
||||
'test/unit/embedding-artifact.test.ts',
|
||||
'test/unit/embedding-job-progress.test.ts',
|
||||
// Descriptor validation differs when O_NOFOLLOW is unavailable on Windows.
|
||||
'test/unit/embedding-job-file-security.test.ts',
|
||||
'test/unit/shared-store-read-security.test.ts',
|
||||
'test/unit/setup.test.ts',
|
||||
'test/unit/setup-jsonc.test.ts',
|
||||
'test/unit/setup-codex.test.ts',
|
||||
|
|
@ -164,6 +173,17 @@ const PLATFORM_LOGIC = [
|
|||
// N-API addon which has known platform-specific behavior (Windows
|
||||
// file-lock lag after close, macOS N-API destructor segfaults)
|
||||
const LBUG_NATIVE = [
|
||||
'test/integration/analysis-embedding-flow.test.ts',
|
||||
'test/integration/analysis-generation-routing.test.ts',
|
||||
'test/integration/analysis-publication-flow.test.ts',
|
||||
'test/integration/embedding-job-controller.test.ts',
|
||||
'test/integration/embedding-job-restart.test.ts',
|
||||
'test/integration/embedding-native-release.test.ts',
|
||||
'test/integration/legacy-embedding-controller.test.ts',
|
||||
'test/integration/legacy-vector-source.test.ts',
|
||||
'test/integration/embedding-vector-store.test.ts',
|
||||
'test/integration/legacy-vector-reuse-index.test.ts',
|
||||
'test/integration/legacy-vector-export.test.ts',
|
||||
'test/integration/java-overload-impact-persistence.test.ts',
|
||||
'test/integration/dangling-owner-edges.test.ts',
|
||||
'test/integration/impact-private-type-file-seed.test.ts',
|
||||
|
|
|
|||
|
|
@ -127,6 +127,8 @@ export interface AnalyzeOptions {
|
|||
* Integer, minimum 200; CLI-only (not a `.gitnexusrc` key).
|
||||
*/
|
||||
memoryBudget?: string;
|
||||
/** Total process-family reservation, separate from the main V8 heap flag. */
|
||||
resourceBudget?: string;
|
||||
/** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */
|
||||
workers?: string;
|
||||
/** Process-detection process cap. Positive integer string; `0` is invalid. */
|
||||
|
|
|
|||
197
gitnexus/src/cli/analyze-phase-ipc.ts
Normal file
197
gitnexus/src/cli/analyze-phase-ipc.ts
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
/** Bounded CLI graph handoff. Live graph consumers run before this is sent. */
|
||||
import path from 'node:path';
|
||||
import type { AnalyzeResult } from '../core/run-analyze.js';
|
||||
import {
|
||||
projectPreparedAnalysisHandoff,
|
||||
type PreparedAnalysisHandoff,
|
||||
} from '../core/analysis-phase-supervisor.js';
|
||||
import { FTS_SKIP_REASONS } from '../core/search/fts-policy.js';
|
||||
|
||||
export interface CliPreparedAnalysisMessage {
|
||||
type: 'graph-prepared';
|
||||
handoff: PreparedAnalysisHandoff;
|
||||
result: Pick<
|
||||
AnalyzeResult,
|
||||
| 'repoPath'
|
||||
| 'repoName'
|
||||
| 'storagePath'
|
||||
| 'stats'
|
||||
| 'ftsSkipped'
|
||||
| 'ftsSkipReason'
|
||||
| 'isPrimaryBranch'
|
||||
>;
|
||||
startedAt: number;
|
||||
/** Snapshot from BEFORE graph/context generation, never recomputed on resume. */
|
||||
selfCommitSafety?: { agentsMd: boolean; claudeMd: boolean };
|
||||
}
|
||||
|
||||
export interface CliAnalysisContinuation {
|
||||
type: 'continue-analysis';
|
||||
writerPid: number;
|
||||
prepared: CliPreparedAnalysisMessage;
|
||||
}
|
||||
|
||||
/** Validate both directions; never retain extra fields from an IPC object. */
|
||||
export function parseCliPreparedAnalysis(value: unknown): CliPreparedAnalysisMessage {
|
||||
const input = value as CliPreparedAnalysisMessage;
|
||||
if (!input || input.type !== 'graph-prepared' || !input.result)
|
||||
throw new Error('Invalid CLI graph phase message');
|
||||
const safety = input.selfCommitSafety;
|
||||
if (safety && (typeof safety.agentsMd !== 'boolean' || typeof safety.claudeMd !== 'boolean'))
|
||||
throw new Error('Invalid CLI self-commit snapshot');
|
||||
return projectCliPreparedAnalysis(
|
||||
{ ...input.result, preparedGraph: input.handoff, rebuildReasons: [] },
|
||||
input.handoff.repoPath,
|
||||
input.startedAt,
|
||||
safety
|
||||
? new Map([
|
||||
['AGENTS.md', safety.agentsMd],
|
||||
['CLAUDE.md', safety.claudeMd],
|
||||
])
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
|
||||
export async function receiveCliAnalysisContinuation(
|
||||
signal?: AbortSignal,
|
||||
): Promise<CliAnalysisContinuation> {
|
||||
signal?.throwIfAborted();
|
||||
if (!process.connected || !process.send)
|
||||
throw new Error('Analysis continuation requires its connected supervisor');
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
process.removeListener('message', onMessage);
|
||||
process.removeListener('disconnect', onDisconnect);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
reject(signal?.reason ?? new Error('Analysis cancelled'));
|
||||
};
|
||||
const onDisconnect = () => {
|
||||
cleanup();
|
||||
reject(new Error('Analysis supervisor disconnected'));
|
||||
};
|
||||
const onMessage = (message: unknown) => {
|
||||
cleanup();
|
||||
try {
|
||||
const input = message as CliAnalysisContinuation;
|
||||
if (
|
||||
input?.type !== 'continue-analysis' ||
|
||||
!Number.isSafeInteger(input.writerPid) ||
|
||||
input.writerPid < 1
|
||||
)
|
||||
throw new Error('Invalid CLI continuation');
|
||||
resolve({
|
||||
type: 'continue-analysis',
|
||||
writerPid: input.writerPid,
|
||||
prepared: parseCliPreparedAnalysis(input.prepared),
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error('Analysis supervisor did not send its continuation'));
|
||||
}, 30_000);
|
||||
process.once('message', onMessage);
|
||||
process.once('disconnect', onDisconnect);
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
process.send!({ type: 'analysis-continuation-ready' }, (error) => {
|
||||
if (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function projectCliPreparedAnalysis(
|
||||
result: AnalyzeResult,
|
||||
repoPath: string,
|
||||
startedAt: number,
|
||||
selfCommitSafety?: ReadonlyMap<string, boolean>,
|
||||
): CliPreparedAnalysisMessage {
|
||||
const handoff = projectPreparedAnalysisHandoff(result.preparedGraph);
|
||||
if (
|
||||
result.graphWriteCollapsed ||
|
||||
result.alreadyUpToDate ||
|
||||
result.ftsRepairedOnly ||
|
||||
result.repoName !== handoff.repoName ||
|
||||
path.resolve(result.repoPath) !== path.resolve(repoPath) ||
|
||||
path.resolve(result.repoPath) !== path.resolve(handoff.repoPath) ||
|
||||
path.resolve(result.storagePath) !== path.resolve(handoff.storagePath)
|
||||
)
|
||||
throw new Error('Prepared CLI graph does not match the requested analysis');
|
||||
if (!Number.isSafeInteger(startedAt) || startedAt < 0)
|
||||
throw new Error('Invalid graph phase start time');
|
||||
|
||||
// Never spread stats or the result: either may gain corpus-sized fields.
|
||||
const stats: AnalyzeResult['stats'] = {};
|
||||
for (const key of [
|
||||
'files',
|
||||
'nodes',
|
||||
'edges',
|
||||
'communities',
|
||||
'processes',
|
||||
'embeddings',
|
||||
] as const) {
|
||||
const value = result.stats[key];
|
||||
if (value === undefined) continue;
|
||||
if (!Number.isSafeInteger(value) || value < 0)
|
||||
throw new Error(`Invalid graph phase counter: ${key}`);
|
||||
stats[key] = value;
|
||||
}
|
||||
for (const key of ['ftsSkipped', 'isPrimaryBranch'] as const) {
|
||||
if (result[key] !== undefined && typeof result[key] !== 'boolean')
|
||||
throw new Error(`Invalid graph phase flag: ${key}`);
|
||||
}
|
||||
if (result.ftsSkipReason !== undefined && !FTS_SKIP_REASONS.includes(result.ftsSkipReason))
|
||||
throw new Error('Invalid graph phase FTS reason');
|
||||
return {
|
||||
type: 'graph-prepared',
|
||||
handoff,
|
||||
result: {
|
||||
repoPath: handoff.repoPath,
|
||||
repoName: handoff.repoName,
|
||||
storagePath: handoff.storagePath,
|
||||
stats,
|
||||
ftsSkipped: result.ftsSkipped,
|
||||
ftsSkipReason: result.ftsSkipReason,
|
||||
isPrimaryBranch: result.isPrimaryBranch,
|
||||
},
|
||||
startedAt,
|
||||
selfCommitSafety: selfCommitSafety
|
||||
? {
|
||||
agentsMd: selfCommitSafety.get('AGENTS.md') === true,
|
||||
claudeMd: selfCommitSafety.get('CLAUDE.md') === true,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
interface SupervisorChannel {
|
||||
connected?: boolean;
|
||||
send?: (message: CliPreparedAnalysisMessage, callback: (error: Error | null) => void) => unknown;
|
||||
}
|
||||
|
||||
export function requireCliAnalysisSupervisor(channel: SupervisorChannel = process) {
|
||||
if (typeof channel.send !== 'function' || channel.connected !== true)
|
||||
throw new Error('Prepared graph requires a connected analysis supervisor');
|
||||
return channel.send.bind(channel);
|
||||
}
|
||||
|
||||
/** A successful send is only delivery, never graph publication or process exit. */
|
||||
export async function sendCliPreparedAnalysis(
|
||||
message: CliPreparedAnalysisMessage,
|
||||
channel: SupervisorChannel = process,
|
||||
): Promise<void> {
|
||||
const send = requireCliAnalysisSupervisor(channel);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
// Await the callback before the caller exits. A false return means channel
|
||||
// backpressure, not failure; the callback remains the delivery boundary.
|
||||
send(message, (error: Error | null) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
/** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs/promises';
|
||||
import { getHeapStatistics } from 'node:v8';
|
||||
import { watchFile, unwatchFile } from 'node:fs';
|
||||
import { watch, type FSWatcher } from 'chokidar';
|
||||
import { createWatchIgnorePredicate } from '../config/ignore-service.js';
|
||||
|
|
@ -23,9 +24,14 @@ import {
|
|||
} from './analyze-config.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
import type { AnalyzeOptions } from './analyze-options.js';
|
||||
import { ensureHeap } from './analyze.js';
|
||||
import { ensureHeap, usesManualAnalysisHeap } from './analyze.js';
|
||||
import { cliError, cliInfo, cliWarn } from './cli-message.js';
|
||||
import { parseIntegerOption } from './int-option.js';
|
||||
import {
|
||||
analysisMayEmbed,
|
||||
analysisSupervisorReservation,
|
||||
resolveHostAnalysisResources,
|
||||
} from '../core/analysis-host-resources.js';
|
||||
import {
|
||||
formatInvalidProcessDetectionOverride,
|
||||
parseProcessDetectionBudgetStrings,
|
||||
|
|
@ -188,6 +194,18 @@ export async function resolveWatchOptions(
|
|||
cliWarn(formatInvalidProcessDetectionOverride(flag, raw));
|
||||
},
|
||||
);
|
||||
const resources = resolveHostAnalysisResources({
|
||||
totalMb: positiveInteger(merged.resourceBudget, '--resource-budget'),
|
||||
mainHeapMb:
|
||||
positiveInteger(cli.memoryBudget, '--memory-budget') ??
|
||||
Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2),
|
||||
workers: workerPoolSize,
|
||||
includeEmbeddings: await analysisMayEmbed(repoPath, { branch }),
|
||||
supervisorBytes:
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined
|
||||
? analysisSupervisorReservation()
|
||||
: Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES),
|
||||
});
|
||||
|
||||
setEnvironment(
|
||||
'GITNEXUS_MAX_FILE_SIZE',
|
||||
|
|
@ -201,6 +219,8 @@ export async function resolveWatchOptions(
|
|||
setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose);
|
||||
|
||||
return {
|
||||
resources,
|
||||
graphPublication: 'auto',
|
||||
pdg: merged.pdg,
|
||||
skipFts: merged.skipFts,
|
||||
branch,
|
||||
|
|
@ -453,182 +473,220 @@ export async function watchCommandWithRunnerIdentity(
|
|||
inputPath?: string,
|
||||
cliOptions: WatchCliOptions = {},
|
||||
): Promise<void> {
|
||||
if (
|
||||
await ensureHeap({ cleanForwardedTermination: true, memoryBudget: cliOptions.memoryBudget })
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd());
|
||||
if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) {
|
||||
cliError(' gitnexus analyze --watch requires a Git repository.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const repoPath = await fs.realpath(requestedRepoPath);
|
||||
try {
|
||||
await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS);
|
||||
} catch (error) {
|
||||
cliError(` ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const baselineEnvironment: WatchEnvironmentBaseline = {
|
||||
maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE,
|
||||
workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS,
|
||||
verbose: process.env.GITNEXUS_VERBOSE,
|
||||
let cancelled = false;
|
||||
let stopWatching: (() => void) | undefined;
|
||||
const cancel = () => {
|
||||
cancelled = true;
|
||||
stopWatching?.();
|
||||
};
|
||||
const onControlMessage = (message: unknown) => {
|
||||
if ((message as { type?: unknown } | null)?.type === 'cancel') cancel();
|
||||
};
|
||||
process.on('message', onControlMessage);
|
||||
process.once('disconnect', cancel);
|
||||
try {
|
||||
let ignoredConfigSignature: string | undefined;
|
||||
const reportIgnoredConfig = (names: readonly string[]) => {
|
||||
const signature = [...names].sort().join(',');
|
||||
if (signature === ignoredConfigSignature) return;
|
||||
ignoredConfigSignature = signature;
|
||||
if (names.length > 0) {
|
||||
cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`);
|
||||
}
|
||||
};
|
||||
let debounceMs: number;
|
||||
let analyzeOptions: CoreAnalyzeOptions;
|
||||
if (process.send) {
|
||||
if (!process.connected) cancel();
|
||||
else
|
||||
process.send({ type: 'analysis-watch-ready' }, (error) => {
|
||||
if (error) cancel();
|
||||
});
|
||||
}
|
||||
const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd());
|
||||
if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) {
|
||||
cliError(' gitnexus analyze --watch requires a Git repository.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const repoPath = await fs.realpath(requestedRepoPath);
|
||||
try {
|
||||
debounceMs =
|
||||
positiveInteger(
|
||||
cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS),
|
||||
'--debounce',
|
||||
MAX_TIMER_DELAY_MS,
|
||||
) ?? DEFAULT_DEBOUNCE_MS;
|
||||
analyzeOptions = await resolveWatchOptions(
|
||||
repoPath,
|
||||
cliOptions,
|
||||
baselineEnvironment,
|
||||
reportIgnoredConfig,
|
||||
);
|
||||
const manualHeap = usesManualAnalysisHeap(cliOptions.memoryBudget);
|
||||
const admission = resolveHostAnalysisResources({
|
||||
totalMb: positiveInteger(cliOptions.resourceBudget, '--resource-budget'),
|
||||
mainHeapMb: manualHeap
|
||||
? Math.ceil(getHeapStatistics().heap_size_limit / 1024 ** 2)
|
||||
: positiveInteger(cliOptions.memoryBudget, '--memory-budget'),
|
||||
workers: positiveInteger(cliOptions.workers, '--workers'),
|
||||
includeEmbeddings: await analysisMayEmbed(repoPath, { branch: cliOptions.branch }),
|
||||
supervisorBytes:
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined
|
||||
? analysisSupervisorReservation()
|
||||
: Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES),
|
||||
});
|
||||
if (cancelled) return;
|
||||
if (
|
||||
!manualHeap &&
|
||||
(await ensureHeap({
|
||||
cleanForwardedTermination: true,
|
||||
memoryBudget: cliOptions.memoryBudget,
|
||||
admittedHeapMb: admission.mainHeapMb,
|
||||
}))
|
||||
)
|
||||
return;
|
||||
await requireStoragePath(repoPath, ANALYZE_STORAGE_REQUIREMENTS);
|
||||
} catch (error) {
|
||||
cliError(` ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
let stopWatching!: () => void;
|
||||
const stopped = new Promise<void>((resolve) => {
|
||||
stopWatching = resolve;
|
||||
});
|
||||
const stop = () => stopWatching();
|
||||
process.once('SIGINT', stop);
|
||||
process.once('SIGTERM', stop);
|
||||
const baselineEnvironment: WatchEnvironmentBaseline = {
|
||||
maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE,
|
||||
workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS,
|
||||
verbose: process.env.GITNEXUS_VERBOSE,
|
||||
};
|
||||
try {
|
||||
let loop: WatchFileLoop;
|
||||
let fatalRefreshError: unknown;
|
||||
let configControlValid = true;
|
||||
let lastSuccessfulRefreshAt: string | undefined;
|
||||
let ignoredConfigSignature: string | undefined;
|
||||
const reportIgnoredConfig = (names: readonly string[]) => {
|
||||
const signature = [...names].sort().join(',');
|
||||
if (signature === ignoredConfigSignature) return;
|
||||
ignoredConfigSignature = signature;
|
||||
if (names.length > 0) {
|
||||
cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`);
|
||||
}
|
||||
};
|
||||
let debounceMs: number;
|
||||
let analyzeOptions: CoreAnalyzeOptions;
|
||||
try {
|
||||
loop = await startWatchFileLoop(
|
||||
debounceMs =
|
||||
positiveInteger(
|
||||
cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS),
|
||||
'--debounce',
|
||||
MAX_TIMER_DELAY_MS,
|
||||
) ?? DEFAULT_DEBOUNCE_MS;
|
||||
analyzeOptions = await resolveWatchOptions(
|
||||
repoPath,
|
||||
debounceMs,
|
||||
async (paths) => {
|
||||
// Initial validation precedes watcher setup. Read again after it is
|
||||
// armed so a change during startup cannot leave stale options live.
|
||||
if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) {
|
||||
const retryingInvalidConfig = !configControlValid;
|
||||
try {
|
||||
analyzeOptions = await resolveWatchOptions(
|
||||
repoPath,
|
||||
cliOptions,
|
||||
baselineEnvironment,
|
||||
reportIgnoredConfig,
|
||||
);
|
||||
configControlValid = true;
|
||||
} catch (error) {
|
||||
configControlValid = false;
|
||||
throw new WatchControlReloadError(
|
||||
retryingInvalidConfig
|
||||
? new Error(
|
||||
'Configuration remains invalid; fix it before indexing more changes.',
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
)
|
||||
: error,
|
||||
);
|
||||
}
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const result = await runFullAnalysis(
|
||||
repoPath,
|
||||
analyzeOptions,
|
||||
{
|
||||
onProgress: () => {},
|
||||
onLog:
|
||||
process.env.GITNEXUS_VERBOSE === '1'
|
||||
? (message) => cliInfo(` ${message}`)
|
||||
: undefined,
|
||||
},
|
||||
runnerIdentityAtBootstrap,
|
||||
);
|
||||
lastSuccessfulRefreshAt = new Date().toISOString();
|
||||
if (paths.length === 0) {
|
||||
cliInfo(
|
||||
result.alreadyUpToDate
|
||||
? `Watching ${repoPath}; index is up to date.`
|
||||
: `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`,
|
||||
);
|
||||
} else {
|
||||
cliInfo(
|
||||
refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt),
|
||||
);
|
||||
}
|
||||
},
|
||||
(error, paths) => {
|
||||
const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : '';
|
||||
if (shouldStopAfterWatchRefreshFailure(error, paths)) {
|
||||
fatalRefreshError = error;
|
||||
const guardTimeout = isIndexLockGuardTimeout(error);
|
||||
cliError(
|
||||
`Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
(guardTimeout
|
||||
? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.'
|
||||
: 'Watch mode is stopping because the live index may have been updated in place.'),
|
||||
guardTimeout
|
||||
? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath }
|
||||
: undefined,
|
||||
);
|
||||
stopWatching();
|
||||
return;
|
||||
}
|
||||
const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet';
|
||||
cliWarn(
|
||||
`Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
`Retry scheduled; last success ${lastSuccess}.`,
|
||||
);
|
||||
},
|
||||
(error) => {
|
||||
fatalRefreshError = error;
|
||||
cliError(
|
||||
`Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
'Watch mode is stopping.',
|
||||
);
|
||||
stopWatching();
|
||||
},
|
||||
process.env[GITNEXUS_RC_ENV]?.trim() || undefined,
|
||||
cliOptions,
|
||||
baselineEnvironment,
|
||||
reportIgnoredConfig,
|
||||
);
|
||||
} catch (error) {
|
||||
cliError(
|
||||
` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
cliError(` ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await stopped;
|
||||
await loop.close();
|
||||
if (fatalRefreshError !== undefined) process.exitCode = 1;
|
||||
if (cancelled) return;
|
||||
const stopped = new Promise<void>((resolve) => {
|
||||
stopWatching = resolve;
|
||||
});
|
||||
const stop = cancel;
|
||||
process.once('SIGINT', stop);
|
||||
process.once('SIGTERM', stop);
|
||||
try {
|
||||
let loop: WatchFileLoop;
|
||||
let fatalRefreshError: unknown;
|
||||
let configControlValid = true;
|
||||
let lastSuccessfulRefreshAt: string | undefined;
|
||||
try {
|
||||
loop = await startWatchFileLoop(
|
||||
repoPath,
|
||||
debounceMs,
|
||||
async (paths) => {
|
||||
if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) {
|
||||
const retryingInvalidConfig = !configControlValid;
|
||||
try {
|
||||
analyzeOptions = await resolveWatchOptions(
|
||||
repoPath,
|
||||
cliOptions,
|
||||
baselineEnvironment,
|
||||
reportIgnoredConfig,
|
||||
);
|
||||
configControlValid = true;
|
||||
} catch (error) {
|
||||
configControlValid = false;
|
||||
throw new WatchControlReloadError(
|
||||
retryingInvalidConfig
|
||||
? new Error(
|
||||
'Configuration remains invalid; fix it before indexing more changes.',
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
)
|
||||
: error,
|
||||
);
|
||||
}
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const result = await runFullAnalysis(
|
||||
repoPath,
|
||||
analyzeOptions,
|
||||
{
|
||||
onProgress: () => {},
|
||||
onLog:
|
||||
process.env.GITNEXUS_VERBOSE === '1'
|
||||
? (message) => cliInfo(` ${message}`)
|
||||
: undefined,
|
||||
},
|
||||
runnerIdentityAtBootstrap,
|
||||
);
|
||||
lastSuccessfulRefreshAt = new Date().toISOString();
|
||||
if (paths.length === 0) {
|
||||
cliInfo(
|
||||
result.alreadyUpToDate
|
||||
? `Watching ${repoPath}; index is up to date.`
|
||||
: `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`,
|
||||
);
|
||||
} else {
|
||||
cliInfo(
|
||||
refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt),
|
||||
);
|
||||
}
|
||||
},
|
||||
(error, paths) => {
|
||||
const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : '';
|
||||
if (shouldStopAfterWatchRefreshFailure(error, paths)) {
|
||||
fatalRefreshError = error;
|
||||
const guardTimeout = isIndexLockGuardTimeout(error);
|
||||
cliError(
|
||||
`Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
(guardTimeout
|
||||
? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.'
|
||||
: 'Watch mode is stopping because the live index may have been updated in place.'),
|
||||
guardTimeout
|
||||
? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath }
|
||||
: undefined,
|
||||
);
|
||||
stopWatching();
|
||||
return;
|
||||
}
|
||||
const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet';
|
||||
cliWarn(
|
||||
`Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
`Retry scheduled; last success ${lastSuccess}.`,
|
||||
);
|
||||
},
|
||||
(error) => {
|
||||
fatalRefreshError = error;
|
||||
cliError(
|
||||
`Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
'Watch mode is stopping.',
|
||||
);
|
||||
stopWatching();
|
||||
},
|
||||
process.env[GITNEXUS_RC_ENV]?.trim() || undefined,
|
||||
);
|
||||
} catch (error) {
|
||||
cliError(
|
||||
` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await stopped;
|
||||
await loop.close();
|
||||
if (fatalRefreshError !== undefined) process.exitCode = 1;
|
||||
} finally {
|
||||
process.removeListener('SIGINT', stop);
|
||||
process.removeListener('SIGTERM', stop);
|
||||
}
|
||||
} finally {
|
||||
process.removeListener('SIGINT', stop);
|
||||
process.removeListener('SIGTERM', stop);
|
||||
setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize);
|
||||
setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout);
|
||||
setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose);
|
||||
}
|
||||
} finally {
|
||||
setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize);
|
||||
setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout);
|
||||
setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose);
|
||||
process.removeListener('message', onControlMessage);
|
||||
process.removeListener('disconnect', cancel);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ import v8 from 'v8';
|
|||
import cliProgress from 'cli-progress';
|
||||
import { formatAnalyzeFtsSkipSummary } from '../core/search/fts-policy.js';
|
||||
import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js';
|
||||
import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js';
|
||||
import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js';
|
||||
import { causeChain } from '../lib/utils.js';
|
||||
import {
|
||||
|
|
@ -51,7 +50,19 @@ import {
|
|||
GitNexusRcError,
|
||||
} from './analyze-config.js';
|
||||
import type { AnalyzeOptions } from './analyze-options.js';
|
||||
import { runFullAnalysis } from '../core/run-analyze.js';
|
||||
import {
|
||||
parseCliPreparedAnalysis,
|
||||
receiveCliAnalysisContinuation,
|
||||
type CliAnalysisContinuation,
|
||||
type CliPreparedAnalysisMessage,
|
||||
} from './analyze-phase-ipc.js';
|
||||
import { AnalysisSupervisorRequired } from '../core/analysis-supervision.js';
|
||||
import {
|
||||
analysisMayEmbed,
|
||||
analysisSupervisorReservation,
|
||||
resolveHostAnalysisResources,
|
||||
} from '../core/analysis-host-resources.js';
|
||||
import type { AnalysisResourceBudget } from '../core/analysis-resource-budget.js';
|
||||
import { getRuntimeFingerprint } from '../core/platform/capabilities.js';
|
||||
import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
|
||||
import {
|
||||
|
|
@ -369,6 +380,8 @@ const shouldBridgeRespawnProgressTty = (): boolean =>
|
|||
process.stderr.isTTY === true || process.stdout.isTTY === true;
|
||||
|
||||
interface RespawnExit {
|
||||
prepared?: CliPreparedAnalysisMessage;
|
||||
writerPid?: number;
|
||||
status?: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
stdout?: string;
|
||||
|
|
@ -402,33 +415,81 @@ const appendOutputTail = (tail: string, chunk: unknown): string => {
|
|||
const runRespawnedAnalyze = (
|
||||
args: readonly string[],
|
||||
env: NodeJS.ProcessEnv,
|
||||
continuation?: CliAnalysisContinuation,
|
||||
): Promise<RespawnExit> =>
|
||||
new Promise((resolve) => {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
let forwardedSignal: NodeJS.Signals | undefined;
|
||||
let prepared: CliPreparedAnalysisMessage | undefined;
|
||||
let phaseError: string | undefined;
|
||||
let cancelTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const child = spawn(process.execPath, [...args], {
|
||||
stdio: ['inherit', 'pipe', 'pipe'],
|
||||
stdio: ['inherit', 'pipe', 'pipe', 'ipc'],
|
||||
windowsHide: true,
|
||||
env,
|
||||
});
|
||||
const forwardSignal = (signal: NodeJS.Signals): void => {
|
||||
forwardedSignal ??= signal;
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill(signal);
|
||||
// This is a standalone process, not a native worker thread. A wedged
|
||||
// process may be killed after grace; dirty/staged receipts recover it.
|
||||
cancelTimer ??= setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
|
||||
}, 60_000);
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'cancel' }, () => {
|
||||
/* Actual exit remains the release boundary. */
|
||||
});
|
||||
} catch {
|
||||
/* A disconnected child cancels at its next safe point. */
|
||||
}
|
||||
}
|
||||
};
|
||||
const forwardSigint = () => forwardSignal('SIGINT');
|
||||
const forwardSigterm = () => forwardSignal('SIGTERM');
|
||||
const finish = (exit: RespawnExit): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(cancelTimer);
|
||||
process.removeListener('SIGINT', forwardSigint);
|
||||
process.removeListener('SIGTERM', forwardSigterm);
|
||||
resolve({ ...exit, forwardedSignal });
|
||||
resolve({
|
||||
...exit,
|
||||
...(phaseError ? { status: 1, message: phaseError } : {}),
|
||||
forwardedSignal,
|
||||
prepared,
|
||||
writerPid: child.pid,
|
||||
});
|
||||
};
|
||||
|
||||
process.once('SIGINT', forwardSigint);
|
||||
process.once('SIGTERM', forwardSigterm);
|
||||
process.on('SIGINT', forwardSigint);
|
||||
process.on('SIGTERM', forwardSigterm);
|
||||
|
||||
child.on('message', (message: unknown) => {
|
||||
try {
|
||||
const type = (message as { type?: string })?.type;
|
||||
if (type === 'graph-prepared') {
|
||||
if (continuation || prepared) throw new Error('Unexpected repeated graph phase');
|
||||
prepared = parseCliPreparedAnalysis(message);
|
||||
} else if (type === 'analysis-continuation-ready' && continuation) {
|
||||
child.send(continuation, (error) => {
|
||||
if (error) phaseError = error.message;
|
||||
});
|
||||
} else if (
|
||||
(type === 'analysis-ready' || type === 'analysis-watch-ready') &&
|
||||
forwardedSignal
|
||||
) {
|
||||
// Cancellation may arrive while the child is still loading modules.
|
||||
forwardSignal(forwardedSignal);
|
||||
}
|
||||
} catch (error) {
|
||||
// Never kill a native-active child for malformed IPC. Its exit remains
|
||||
// the release boundary and a failed projection cannot start a model.
|
||||
phaseError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
});
|
||||
|
||||
child.stdout?.on('data', (chunk) => {
|
||||
stdout = appendOutputTail(stdout, chunk);
|
||||
|
|
@ -544,6 +605,18 @@ const forceHeapOOMForTestIfEnabled = (): void => {
|
|||
// `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value.
|
||||
const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024;
|
||||
|
||||
/** CLI and watch must preserve the same explicit heap/autopilot opt-outs. */
|
||||
export function usesManualAnalysisHeap(memoryBudget?: string): boolean {
|
||||
return (
|
||||
memoryBudget === undefined &&
|
||||
(process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP === '1' ||
|
||||
(process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' &&
|
||||
(memoryAutopilotDisabled() ||
|
||||
parseMaxOldSpaceMb(process.execArgv.join(' ')) !== null ||
|
||||
(parseMaxOldSpaceMb(process.env.NODE_OPTIONS ?? '') ?? 0) >= RESPAWN_HEAP_MB)))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Last `--max-old-space-size` value (MB) in a NODE_OPTIONS string, or `null`
|
||||
* when absent/unparseable. Last occurrence wins, matching V8's own
|
||||
|
|
@ -681,9 +754,42 @@ export function resolveBudgetHeap(input: BudgetHeapInput): BudgetHeapDecision {
|
|||
* respawned) so the parse phase's remedy text advises by source. Explicit
|
||||
* heap pins and `GITNEXUS_MEMORY=off` intentionally leave it unset. */
|
||||
export async function ensureHeap(
|
||||
options: { cleanForwardedTermination?: boolean; memoryBudget?: string } = {},
|
||||
options: {
|
||||
cleanForwardedTermination?: boolean;
|
||||
memoryBudget?: string;
|
||||
admittedHeapMb?: number;
|
||||
} = {},
|
||||
): Promise<boolean> {
|
||||
const nodeOpts = process.env.NODE_OPTIONS || '';
|
||||
if (
|
||||
options.memoryBudget === undefined &&
|
||||
options.admittedHeapMb !== undefined &&
|
||||
!(Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0)
|
||||
) {
|
||||
const decision = resolveBudgetHeap({
|
||||
budgetMb: options.admittedHeapMb,
|
||||
execArgv: process.execArgv,
|
||||
nodeOptions: nodeOpts,
|
||||
autoCapMb: RESPAWN_HEAP_MB,
|
||||
autopilotDisabled: false,
|
||||
inheritedSource: 'budget',
|
||||
});
|
||||
if (!decision.respawn) return false;
|
||||
const ambient = parseMaxOldSpaceMb(nodeOpts);
|
||||
if (ambient !== null && process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1') {
|
||||
cliWarn(
|
||||
` NODE_OPTIONS pins the heap to ${ambient}MB. Re-running analyze with the admitted ${options.admittedHeapMb}MB main heap.\n`,
|
||||
);
|
||||
}
|
||||
return respawnWithHeap(
|
||||
`--max-old-space-size=${decision.sizing.oldSpaceMb}`,
|
||||
`--max-semi-space-size=${decision.sizing.semiSpaceMb}`,
|
||||
'auto',
|
||||
'Analysis likely ran out of memory in its admitted main heap. Raise --resource-budget or reduce the analysis scope. If this persists, it may be a native crash unrelated to heap size.',
|
||||
nodeOpts,
|
||||
options,
|
||||
);
|
||||
}
|
||||
// The budget is checked BEFORE the GITNEXUS_MEMORY=off return: an explicit
|
||||
// flag is a deliberate per-run choice, which the opt-out does not cover.
|
||||
if (options.memoryBudget !== undefined) {
|
||||
|
|
@ -766,16 +872,16 @@ export async function ensureHeap(
|
|||
|
||||
/** Run the analyze child with the given heap flags and map its exit onto this process. */
|
||||
async function respawnWithHeap(
|
||||
heapFlag: string,
|
||||
semiFlag: string,
|
||||
source: HeapLimitSource,
|
||||
heapFlag: string | undefined,
|
||||
semiFlag: string | undefined,
|
||||
source: HeapLimitSource | undefined,
|
||||
oomGuidance: string,
|
||||
nodeOpts: string,
|
||||
options: { cleanForwardedTermination?: boolean },
|
||||
): Promise<boolean> {
|
||||
// --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it
|
||||
// only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS.
|
||||
const cliFlags = [heapFlag, semiFlag];
|
||||
const cliFlags = [heapFlag, semiFlag].filter((flag): flag is string => flag !== undefined);
|
||||
if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG);
|
||||
|
||||
// Preserve the parent's node flags (execArgv) — dropping them breaks any
|
||||
|
|
@ -789,11 +895,28 @@ async function respawnWithHeap(
|
|||
const childArgs = [...preservedExecArgv, ...cliFlags, ...process.argv.slice(1)];
|
||||
const childEnv = {
|
||||
...process.env,
|
||||
NODE_OPTIONS: `${nodeOpts} ${heapFlag} ${semiFlag}`.trim(),
|
||||
NODE_OPTIONS: [nodeOpts, heapFlag, semiFlag].filter(Boolean).join(' ').trim(),
|
||||
[HEAP_LIMIT_SOURCE_ENV]: source,
|
||||
GITNEXUS_ANALYSIS_SUPERVISED: '1',
|
||||
GITNEXUS_ANALYSIS_MANUAL_HEAP: heapFlag === undefined ? '1' : '0',
|
||||
GITNEXUS_ANALYSIS_SUPERVISOR_BYTES:
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES ?? String(analysisSupervisorReservation()),
|
||||
};
|
||||
if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1';
|
||||
const childExit = await runRespawnedAnalyze(childArgs, childEnv);
|
||||
let childExit = await runRespawnedAnalyze(childArgs, childEnv);
|
||||
if (
|
||||
childExit.status === 0 &&
|
||||
!childExit.signal &&
|
||||
!childExit.forwardedSignal &&
|
||||
childExit.prepared
|
||||
) {
|
||||
if (!childExit.writerPid) throw new Error('Graph process did not report its PID');
|
||||
childExit = await runRespawnedAnalyze(
|
||||
childArgs,
|
||||
{ ...childEnv, GITNEXUS_ANALYSIS_CONTINUATION: '1' },
|
||||
{ type: 'continue-analysis', writerPid: childExit.writerPid, prepared: childExit.prepared },
|
||||
);
|
||||
}
|
||||
if (childExit.forwardedSignal !== undefined) {
|
||||
process.exitCode = forwardedSignalExitCode(
|
||||
childExit.forwardedSignal,
|
||||
|
|
@ -830,6 +953,8 @@ async function respawnWithHeap(
|
|||
* for it in the first place.
|
||||
*/
|
||||
const ANALYZE_CLI_ENV_KEYS = [
|
||||
'GITNEXUS_ANALYSIS_MAIN_HEAP_MB',
|
||||
'GITNEXUS_ANALYSIS_SUPERVISOR_BYTES',
|
||||
HEAP_LIMIT_SOURCE_ENV,
|
||||
'GITNEXUS_VERBOSE',
|
||||
'GITNEXUS_PROFILE_DEFERRED',
|
||||
|
|
@ -903,19 +1028,12 @@ export const analyzeCommand = async (
|
|||
// Snapshot before ensureHeap: it records GITNEXUS_HEAP_LIMIT_SOURCE on a
|
||||
// kept process, which must not leak into a later programmatic call.
|
||||
const envSnap = snapshotAnalyzeEnv();
|
||||
if (await ensureHeap({ memoryBudget: options?.memoryBudget })) return;
|
||||
forceHeapOOMForTestIfEnabled();
|
||||
|
||||
// Install fatal handlers immediately after re-exec resolution so any
|
||||
// Install fatal handlers before admission so any
|
||||
// async error that escapes the try/catch below (#1169) surfaces with
|
||||
// a stack trace and a non-zero exit code instead of a silent exit 0.
|
||||
installFatalHandlers();
|
||||
|
||||
// npm-11 npx-crash nudge (#1939). Runs here, after the heap re-exec guard,
|
||||
// so it fires once in the working process and never on the lazy-startup path
|
||||
// of other commands (e.g. `gitnexus mcp`).
|
||||
warnIfNpm11NpxRisk();
|
||||
|
||||
// Snapshot the GITNEXUS_* env vars that the impl writes for downstream
|
||||
// consumption, so they don't leak across `analyzeCommand` invocations in
|
||||
// programmatic callers (tests, long-running hosts). `process.exit(0)` on
|
||||
|
|
@ -1285,6 +1403,61 @@ const analyzeCommandImpl = async (
|
|||
process.env.GITNEXUS_EMBEDDING_DIMS = dims;
|
||||
}
|
||||
|
||||
let resources: AnalysisResourceBudget;
|
||||
let manualHeap = false;
|
||||
try {
|
||||
const supervisorBytes =
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES === undefined
|
||||
? analysisSupervisorReservation()
|
||||
: Number(process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES);
|
||||
manualHeap = usesManualAnalysisHeap(options.memoryBudget);
|
||||
resources = resolveHostAnalysisResources({
|
||||
totalMb:
|
||||
options.resourceBudget === undefined
|
||||
? undefined
|
||||
: parseIntegerOption(options.resourceBudget, '--resource-budget', { minimum: 1 }),
|
||||
mainHeapMb:
|
||||
options.memoryBudget !== undefined
|
||||
? parseMemoryBudgetMb(options.memoryBudget)
|
||||
: process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1' &&
|
||||
process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB !== undefined
|
||||
? Number(process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB)
|
||||
: manualHeap
|
||||
? Math.ceil(v8.getHeapStatistics().heap_size_limit / 1024 ** 2)
|
||||
: undefined,
|
||||
workers: workerPoolSize,
|
||||
includeEmbeddings: await analysisMayEmbed(repoPath, {
|
||||
embeddings: embeddingsEnabled,
|
||||
force: options.force,
|
||||
dropEmbeddings: options.dropEmbeddings,
|
||||
branch: cliOptions?.branch,
|
||||
}),
|
||||
supervisorBytes,
|
||||
});
|
||||
// The two child phases retain the same frontend and heap reservation.
|
||||
// Re-measuring either in the child would add another supervisor or trigger
|
||||
// a nested heap respawn with a slightly different allocation.
|
||||
process.env.GITNEXUS_ANALYSIS_MAIN_HEAP_MB = String(resources.mainHeapMb);
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISOR_BYTES = String(supervisorBytes);
|
||||
if (
|
||||
!manualHeap &&
|
||||
(await ensureHeap({
|
||||
memoryBudget: options.memoryBudget,
|
||||
admittedHeapMb: resources.mainHeapMb,
|
||||
}))
|
||||
)
|
||||
return;
|
||||
forceHeapOOMForTestIfEnabled();
|
||||
} catch (error) {
|
||||
cliError(` ${error instanceof Error ? error.message : String(error)}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// The npm compatibility warning probes subprocesses. Run it only after the
|
||||
// heap re-exec guard. A continuation's graph process already emitted it.
|
||||
if (process.env.GITNEXUS_ANALYSIS_CONTINUATION !== '1') warnIfNpm11NpxRisk();
|
||||
|
||||
// Custom-endpoint UX, emitting at most ONE message that reflects THIS run's
|
||||
// intent (not ambient env). Order matters — the first matching branch wins:
|
||||
// 1. flags given but --embeddings absent: the endpoint won't be used, so
|
||||
|
|
@ -1463,29 +1636,38 @@ const analyzeCommandImpl = async (
|
|||
|
||||
bar.start(100, 0, { phase: 'Initializing...' });
|
||||
|
||||
// Graceful SIGINT handling. Pino's default destination is `sync: false`
|
||||
// (buffered) — flush before exit so in-flight records reach stderr.
|
||||
// See `gitnexus/src/core/logger.ts:flushLoggerSync`.
|
||||
let aborted = false;
|
||||
const sigintHandler = () => {
|
||||
if (aborted) process.exit(1);
|
||||
aborted = true;
|
||||
bar.stop();
|
||||
console.log('\n Interrupted — cleaning up...');
|
||||
// Bounded CHECKPOINT-then-exit (#2264 review P3): skip the native close (the
|
||||
// LadybugDB destructor can double-free after --pdg writes), but don't hang
|
||||
// behind a long --pdg COPY holding the connection lock — bound it so a single
|
||||
// Ctrl-C stays responsive; the WAL replays on the next analyze. A second
|
||||
// Ctrl-C (`if (aborted) process.exit(1)` above) remains the escape hatch.
|
||||
void boundedCheckpointBeforeExit({
|
||||
exitCode: 130,
|
||||
beforeExit: async () => {
|
||||
const { flushLoggerSync } = await import('../core/logger.js');
|
||||
flushLoggerSync();
|
||||
},
|
||||
});
|
||||
// Signals and IPC only request cancellation. The analysis promise owns its
|
||||
// native calls and cleanup; even a repeated interrupt must not exit mid-call.
|
||||
const cancellation = new AbortController();
|
||||
let cancellationExitCode = 130;
|
||||
const requestCancellation = (code: number) => {
|
||||
if (cancellation.signal.aborted) return;
|
||||
cancellationExitCode = code;
|
||||
cancellation.abort(new Error('Analysis cancelled'));
|
||||
console.log('\n Interrupted — waiting for native work to finish...');
|
||||
};
|
||||
const sigintHandler = () => requestCancellation(130);
|
||||
const sigtermHandler = () => requestCancellation(143);
|
||||
const cancelMessage = (message: unknown) => {
|
||||
if ((message as { type?: string })?.type === 'cancel') requestCancellation(130);
|
||||
};
|
||||
const disconnected = () => requestCancellation(1);
|
||||
process.on('SIGINT', sigintHandler);
|
||||
process.on('SIGTERM', sigtermHandler);
|
||||
process.on('message', cancelMessage);
|
||||
process.on('disconnect', disconnected);
|
||||
if (process.env.GITNEXUS_ANALYSIS_SUPERVISED === '1') {
|
||||
if (!process.connected || !process.send) disconnected();
|
||||
else {
|
||||
try {
|
||||
process.send({ type: 'analysis-ready' }, (error) => {
|
||||
if (error) disconnected();
|
||||
});
|
||||
} catch {
|
||||
disconnected();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Route console output through bar.log() to prevent progress bar corruption.
|
||||
// This is a deliberate UI pattern (not a logging concern): analyze runs a
|
||||
|
|
@ -1532,14 +1714,23 @@ const analyzeCommandImpl = async (
|
|||
}
|
||||
}, 1000);
|
||||
|
||||
const t0 = Date.now();
|
||||
|
||||
// ── Run shared analysis orchestrator ───────────────────────────────
|
||||
try {
|
||||
// The heap supervisor never runs the pipeline. Load it only in the
|
||||
// admitted working process, after validation and the re-exec guard.
|
||||
const analysis = await import('../core/run-analyze.js');
|
||||
const continuation =
|
||||
process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1'
|
||||
? await receiveCliAnalysisContinuation(cancellation.signal)
|
||||
: undefined;
|
||||
cancellation.signal.throwIfAborted();
|
||||
const t0 = continuation?.prepared.startedAt ?? Date.now();
|
||||
const skipAll = options.indexOnly;
|
||||
const skipAgentsMd = skipAll || options.skipAgentsMd;
|
||||
const skipSkills = skipAll || options.skipSkills;
|
||||
const runOptions = {
|
||||
graphPublication: 'auto' as const,
|
||||
resources,
|
||||
// The user's own --force only. --skills (skill generation needs a fresh
|
||||
// pipelineResult) and --no-parse-cache (bypassing parser output means
|
||||
// anything only when the pipeline runs) each force the rebuild inside
|
||||
|
|
@ -1604,6 +1795,11 @@ const analyzeCommandImpl = async (
|
|||
skipNativeCloseOnExit: true,
|
||||
};
|
||||
const runCallbacks = {
|
||||
signal: cancellation.signal,
|
||||
onGraphWork: () => {
|
||||
if (process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' || !process.connected)
|
||||
throw new AnalysisSupervisorRequired();
|
||||
},
|
||||
onProgress: (_phase, percent, message) => {
|
||||
updateBar(percent, message);
|
||||
},
|
||||
|
|
@ -1617,11 +1813,45 @@ const analyzeCommandImpl = async (
|
|||
// further down) writes to them, so selfCommitContextFiles can tell a
|
||||
// pre-existing unstaged user edit apart from this run's stats refresh
|
||||
// and refuse to sweep the former into the latter's commit.
|
||||
const selfCommitSafety =
|
||||
options.selfCommit === true
|
||||
const selfCommitSafety = continuation?.prepared.selfCommitSafety
|
||||
? new Map([
|
||||
['AGENTS.md', continuation.prepared.selfCommitSafety.agentsMd],
|
||||
['CLAUDE.md', continuation.prepared.selfCommitSafety.claudeMd],
|
||||
])
|
||||
: options.selfCommit === true
|
||||
? snapshotSelfCommitSafety(repoPath, ['AGENTS.md', 'CLAUDE.md'])
|
||||
: undefined;
|
||||
const result = await runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs);
|
||||
const result = continuation
|
||||
? {
|
||||
...continuation.prepared.result,
|
||||
stats:
|
||||
(
|
||||
await analysis.continuePreparedAnalysis(
|
||||
continuation.prepared.handoff,
|
||||
continuation.writerPid,
|
||||
runOptions,
|
||||
runCallbacks,
|
||||
cancellation.signal,
|
||||
)
|
||||
).stats ?? {},
|
||||
rebuildReasons: [],
|
||||
pipelineResult: undefined,
|
||||
preparedGraph: undefined,
|
||||
alreadyUpToDate: false,
|
||||
ftsRepairedOnly: false,
|
||||
graphWriteCollapsed: undefined,
|
||||
}
|
||||
: await analysis.runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs);
|
||||
cancellation.signal.throwIfAborted();
|
||||
|
||||
let preparedMessage: CliPreparedAnalysisMessage | undefined;
|
||||
if (result.preparedGraph) {
|
||||
const { projectCliPreparedAnalysis, requireCliAnalysisSupervisor } =
|
||||
await import('./analyze-phase-ipc.js');
|
||||
requireCliAnalysisSupervisor();
|
||||
// Validate identity before skill/context writes use the registered name.
|
||||
preparedMessage = projectCliPreparedAnalysis(result, repoPath, t0, selfCommitSafety);
|
||||
}
|
||||
|
||||
if (result.alreadyUpToDate) {
|
||||
// Even the fast path must prove the repo is discoverable. A prior
|
||||
|
|
@ -1701,7 +1931,7 @@ const analyzeCommandImpl = async (
|
|||
// success so the silent-finalize state surfaces with a non-zero
|
||||
// exit code and an actionable error instead of being mistaken for
|
||||
// a healthy index.
|
||||
await assertAnalysisFinalized(repoPath, result.storagePath);
|
||||
if (!preparedMessage) await assertAnalysisFinalized(repoPath, result.storagePath);
|
||||
|
||||
// Skill generation (CLI-only, uses pipeline result from analysis).
|
||||
// Gated so `--index-only --skills` skips community skill writes too
|
||||
|
|
@ -1765,6 +1995,24 @@ const analyzeCommandImpl = async (
|
|||
}
|
||||
}
|
||||
|
||||
if (preparedMessage) {
|
||||
// Skill generation above needs the live graph and stays in this child.
|
||||
// Publication, embeddings, the success banner and self-commit belong to
|
||||
// the supervisor only after this process has actually exited.
|
||||
const { sendCliPreparedAnalysis } = await import('./analyze-phase-ipc.js');
|
||||
await sendCliPreparedAnalysis(preparedMessage);
|
||||
clearInterval(elapsedTimer);
|
||||
process.removeListener('SIGINT', sigintHandler);
|
||||
console.log = origLog;
|
||||
// eslint-disable-next-line no-console -- restoring progress-bar routing
|
||||
console.warn = origWarn;
|
||||
// eslint-disable-next-line no-console -- restoring progress-bar routing
|
||||
console.error = origError;
|
||||
bar.stop();
|
||||
process.exitCode = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
// #2639: opt-in self-commit of any AGENTS.md/CLAUDE.md churn written by
|
||||
// this run (the primary generateAIContextFiles call inside
|
||||
// runFullAnalysis, and/or the --skills regeneration above). Best-effort
|
||||
|
|
@ -1842,6 +2090,31 @@ const analyzeCommandImpl = async (
|
|||
console.error = origError;
|
||||
bar.stop();
|
||||
|
||||
if (cancellation.signal.aborted) {
|
||||
process.exitCode = cancellationExitCode;
|
||||
return;
|
||||
}
|
||||
|
||||
if (err instanceof AnalysisSupervisorRequired) {
|
||||
const sizing = resolveBudgetHeap({
|
||||
budgetMb: resources.mainHeapMb,
|
||||
execArgv: process.execArgv,
|
||||
nodeOptions: process.env.NODE_OPTIONS ?? '',
|
||||
autoCapMb: RESPAWN_HEAP_MB,
|
||||
autopilotDisabled: false,
|
||||
inheritedSource: 'budget',
|
||||
}).sizing;
|
||||
await respawnWithHeap(
|
||||
manualHeap ? undefined : `--max-old-space-size=${sizing.oldSpaceMb}`,
|
||||
manualHeap ? undefined : `--max-semi-space-size=${sizing.semiSpaceMb}`,
|
||||
manualHeap ? (process.env[HEAP_LIMIT_SOURCE_ENV] as HeapLimitSource | undefined) : 'auto',
|
||||
'Analysis exceeded its resource budget.',
|
||||
process.env.NODE_OPTIONS ?? '',
|
||||
{},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
|
||||
// Registry name-collision from --name (#829) — surface as an
|
||||
|
|
@ -2195,6 +2468,12 @@ const analyzeCommandImpl = async (
|
|||
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
} finally {
|
||||
clearInterval(elapsedTimer);
|
||||
process.removeListener('SIGINT', sigintHandler);
|
||||
process.removeListener('SIGTERM', sigtermHandler);
|
||||
process.removeListener('message', cancelMessage);
|
||||
process.removeListener('disconnect', disconnected);
|
||||
}
|
||||
|
||||
// LadybugDB's native module holds open handles that prevent Node from exiting.
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import {
|
|||
} from '../core/lbug/sidecar-recovery.js';
|
||||
import { t } from './i18n/index.js';
|
||||
import { getGlobalDir } from '../storage/global-dir.js';
|
||||
import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js';
|
||||
import { STORES_DIR } from '../storage/shared-store.js';
|
||||
import {
|
||||
findLegacyLocalIndex,
|
||||
|
|
@ -271,8 +272,8 @@ export const cleanCommand = async (options?: {
|
|||
console.log(t('clean.branchNotIndexed', { branch: options.branch }));
|
||||
return;
|
||||
}
|
||||
const { lbugPath } = getStoragePaths(repo.repoPath, summary.branch, storagePath);
|
||||
const branchDir = path.dirname(lbugPath);
|
||||
// Cleanup targets the owned slot, even when its graph pointer is damaged.
|
||||
const branchDir = path.join(storagePath, BRANCHES_DIR, branchSlug(summary.branch));
|
||||
if (!isContainedBranchDir(storagePath, branchDir)) {
|
||||
logger.error(
|
||||
`Refusing to clean branch index outside the validated storage slot: ${branchDir}`,
|
||||
|
|
|
|||
|
|
@ -1,36 +1,16 @@
|
|||
import { lstat } from 'node:fs/promises';
|
||||
import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js';
|
||||
import { LBUG_DIRECTORY } from '../storage/storage-constants.js';
|
||||
import { resolveGraphPath } from '../storage/shared-store.js';
|
||||
import { resolveHostAnalysisResources } from '../core/analysis-host-resources.js';
|
||||
import { readPreparedAnalysis } from '../core/analysis-publication.js';
|
||||
import path from 'node:path';
|
||||
import { cliInfo } from './cli-message.js';
|
||||
import { getGitRoot } from '../storage/git.js';
|
||||
import {
|
||||
acquireIndexLock,
|
||||
requireExclusiveIndexLock,
|
||||
sweepStagingArtifacts,
|
||||
} from '../storage/index-lock.js';
|
||||
import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js';
|
||||
import {
|
||||
closeLbug,
|
||||
executeQuery,
|
||||
executeWithReusedStatement,
|
||||
fetchExistingEmbeddingHashes,
|
||||
initLbug,
|
||||
} from '../core/lbug/lbug-adapter.js';
|
||||
import {
|
||||
decideEmbeddingResume,
|
||||
mintInterruptedCheckpoint,
|
||||
mintPartialCheckpoint,
|
||||
mintUnverifiedCountCheckpoint,
|
||||
type EmbeddingCheckpoint,
|
||||
type EmbeddingCheckpointProgress,
|
||||
} from '../core/embedding-checkpoint.js';
|
||||
import { requireExclusiveIndexLock, sweepStagingArtifacts } from '../storage/index-lock.js';
|
||||
import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js';
|
||||
import { getStoragePaths, loadMeta } from '../storage/repo-manager.js';
|
||||
import { decideEmbeddingResume } from '../core/embedding-checkpoint.js';
|
||||
import { EMBEDDING_DIMS, embeddingDimsMismatch } from '../core/lbug/schema.js';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import {
|
||||
measurePersistedEmbeddingCount,
|
||||
persistedEmbeddingCountOrUndefined,
|
||||
} from '../core/embedding-count.js';
|
||||
import { isHttpMode } from '../core/embeddings/http-client.js';
|
||||
import {
|
||||
ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS,
|
||||
|
|
@ -42,8 +22,6 @@ import {
|
|||
assessLocalEmbeddingRuntime,
|
||||
localEmbeddingStackMissingMessage,
|
||||
} from '../core/embeddings/runtime-support.js';
|
||||
import { reapEmbeddingSidecarSafely } from '../core/embeddings/embedding-sidecar-reap.js';
|
||||
import { contentRetentionFromMeta } from '../core/content-retention.js';
|
||||
|
||||
/** Add missing embeddings directly to a healthy index, checkpointing periodically. */
|
||||
export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> => {
|
||||
|
|
@ -54,8 +32,11 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
const metaDir = path.dirname(metaPath);
|
||||
// Writes go to the slot's own graph. A shared-store checkout that reads an
|
||||
// immutable commit graph (#3352) takes a private copy first.
|
||||
const lbugPath = path.join(metaDir, LBUG_DIRECTORY);
|
||||
const lock = await acquireIndexLock(metaDir, { sweep: false });
|
||||
const resources = resolveHostAnalysisResources({
|
||||
includeParsing: false,
|
||||
includeEmbeddings: true,
|
||||
});
|
||||
const lock = await acquireAnalysisSlotLock(metaDir, { sweep: false });
|
||||
try {
|
||||
requireExclusiveIndexLock(
|
||||
lock,
|
||||
|
|
@ -70,6 +51,8 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
'Run `gitnexus analyze` to recover them first.',
|
||||
);
|
||||
}
|
||||
if (await readPreparedAnalysis(metaDir))
|
||||
throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.');
|
||||
sweepStagingArtifacts(metaDir);
|
||||
if (!(await ensurePrivateSharedGraph(metaDir, (m) => console.log(` ${m}`)))) {
|
||||
throw new Error('The shared graph this checkout reads is gone. Run gitnexus analyze first.');
|
||||
|
|
@ -81,6 +64,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
throw new Error('The structural index is incomplete. Run gitnexus analyze --force first.');
|
||||
}
|
||||
|
||||
const lbugPath = resolveGraphPath(metaDir);
|
||||
let lbugStat;
|
||||
try {
|
||||
lbugStat = await lstat(lbugPath);
|
||||
|
|
@ -97,8 +81,6 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
|
||||
const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js');
|
||||
const identity = resolveEmbeddingIdentity();
|
||||
let forceReembedNodeIds: ReadonlySet<string> | undefined;
|
||||
let resumedFrom: EmbeddingCheckpoint | undefined;
|
||||
if (meta.embeddingCheckpoint) {
|
||||
const checkpoint = meta.embeddingCheckpoint;
|
||||
const decision = decideEmbeddingResume(checkpoint, identity);
|
||||
|
|
@ -125,25 +107,15 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
);
|
||||
}
|
||||
cliInfo(decision.log);
|
||||
if (decision.action === 'resume') {
|
||||
forceReembedNodeIds = decision.pendingNodeIds;
|
||||
resumedFrom = decision.resumedFrom;
|
||||
}
|
||||
}
|
||||
|
||||
// The vector column is FLOAT[N] fixed when the index was built, and the
|
||||
// pipeline deletes each batch's stale rows immediately before inserting the
|
||||
// replacements — so a width change here deletes rows it cannot re-insert.
|
||||
// `analyze` forces a full rebuild on the same mismatch; only a rebuild can
|
||||
// retype the column, so this writer refuses instead. An absent recorded
|
||||
// width is not a mismatch (see `embeddingDimsMismatch`).
|
||||
if (embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) {
|
||||
// Keep legacy sync's explicit migration requirement; the new vector store
|
||||
// can use a different width once analyze has established its own generation.
|
||||
if (!meta.graphGeneration && embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS))
|
||||
throw new Error(
|
||||
`Cannot sync embeddings: this index stores FLOAT[${meta.embeddingDims}] vectors, ` +
|
||||
`but this run embeds at ${EMBEDDING_DIMS} dimensions. ` +
|
||||
'Run `gitnexus analyze --embeddings --force` to rebuild the column at the new width.',
|
||||
`but this run embeds at ${EMBEDDING_DIMS} dimensions. Run gitnexus analyze --embeddings --force.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!isHttpMode()) {
|
||||
const assessment = assessLocalEmbeddingRuntime();
|
||||
|
|
@ -167,93 +139,27 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
|
|||
}
|
||||
}
|
||||
|
||||
await initLbug(lbugPath);
|
||||
try {
|
||||
const existing = await fetchExistingEmbeddingHashes(executeQuery);
|
||||
let lastPercent = -1;
|
||||
|
||||
const countEmbeddings = async (): Promise<number | undefined> =>
|
||||
persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery));
|
||||
// One write path for every meta update this command makes. The re-read
|
||||
// happens immediately before each save so a concurrent writer's fields
|
||||
// survive. #2790 traced two production drifts to hand-copied writers of
|
||||
// these exact fields, so this file keeps one copy instead of three.
|
||||
const persistMeta = async (patch: (latest: RepoMeta) => Partial<RepoMeta>): Promise<void> => {
|
||||
const latest = (await loadMeta(metaDir)) ?? meta;
|
||||
await saveMeta(metaDir, { ...latest, ...patch(latest) });
|
||||
};
|
||||
const saveCheckpoint = async (
|
||||
checkpoint: EmbeddingCheckpointProgress,
|
||||
pendingNodeIds: string[],
|
||||
embeddings?: number,
|
||||
): Promise<void> =>
|
||||
persistMeta((latest) => ({
|
||||
...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }),
|
||||
embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds),
|
||||
}));
|
||||
|
||||
cliInfo(`Embedding ${repoPath}`);
|
||||
cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`);
|
||||
|
||||
const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
|
||||
const result = await runEmbeddingPipeline(
|
||||
executeQuery,
|
||||
executeWithReusedStatement,
|
||||
(progress) => {
|
||||
const percent = Math.floor(progress.percent);
|
||||
if (percent !== lastPercent && (percent % 5 === 0 || percent === 100)) {
|
||||
lastPercent = percent;
|
||||
cliInfo(
|
||||
` ${percent}% — ${progress.nodesProcessed ?? 0}/${progress.totalNodes ?? '?'} nodes`,
|
||||
);
|
||||
cliInfo(`Embedding ${repoPath}`);
|
||||
let lastPercent = -1;
|
||||
const { runAnalysisEmbeddingPhase } = await import('../core/analysis-embedding-phase.js');
|
||||
const completed = await runAnalysisEmbeddingPhase({
|
||||
metadataDirectory: metaDir,
|
||||
graphPath: lbugPath,
|
||||
resources,
|
||||
nodeLimit: 0,
|
||||
expectedGeneration: meta.graphGeneration?.id,
|
||||
callbacks: {
|
||||
onLog: cliInfo,
|
||||
onProgress: (_phase, percent, message) => {
|
||||
const rounded = Math.floor(percent);
|
||||
if (rounded !== lastPercent) {
|
||||
lastPercent = rounded;
|
||||
cliInfo(` ${rounded}% — ${message}`);
|
||||
}
|
||||
},
|
||||
{},
|
||||
undefined,
|
||||
existing && existing.size ? existing : undefined,
|
||||
{
|
||||
source: {
|
||||
repoPath,
|
||||
fileHashes: meta.fileHashes,
|
||||
contentRetention: contentRetentionFromMeta(meta),
|
||||
},
|
||||
forceReembedNodeIds,
|
||||
onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => {
|
||||
await saveCheckpoint(checkpoint, nodeIds);
|
||||
},
|
||||
onCheckpoint: async (checkpoint) => {
|
||||
await saveCheckpoint(checkpoint, [], await countEmbeddings());
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const embeddings = await countEmbeddings();
|
||||
if (embeddings === undefined) {
|
||||
// Keep last-known stats.embeddings. An interrupted window marker would
|
||||
// fail the identity gate on the next run even though this run finished;
|
||||
// unverified-count is the recovery kind that forces a recount (#2790).
|
||||
await persistMeta(() => ({
|
||||
embeddingCheckpoint: result.failedNodeIds.length
|
||||
? mintPartialCheckpoint(identity, result, resumedFrom)
|
||||
: mintUnverifiedCountCheckpoint(identity, {
|
||||
nodesProcessed: result.nodesProcessed,
|
||||
totalNodes: result.nodesProcessed,
|
||||
chunksProcessed: result.chunksProcessed,
|
||||
}),
|
||||
}));
|
||||
throw new Error('Could not verify persisted embedding count.');
|
||||
}
|
||||
await persistMeta((latest) => ({
|
||||
stats: { ...latest.stats, embeddings },
|
||||
embeddingCheckpoint: result.failedNodeIds.length
|
||||
? mintPartialCheckpoint(identity, result, resumedFrom)
|
||||
: undefined,
|
||||
}));
|
||||
cliInfo(`Embeddings ready: ${embeddings}`);
|
||||
} finally {
|
||||
await closeLbug().catch(() => {});
|
||||
await reapEmbeddingSidecarSafely();
|
||||
}
|
||||
},
|
||||
});
|
||||
cliInfo(`Embeddings ready: ${completed.stats?.embeddings ?? 0}`);
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ const OPTION_DESCRIPTION_KEYS = {
|
|||
'analyze|--worker-timeout <seconds>': 'help.option.analyze.workerTimeout',
|
||||
'analyze|--wal-checkpoint-threshold <bytes>': 'help.option.analyze.walCheckpointThreshold',
|
||||
'analyze|--memory-budget <mb>': 'help.option.analyze.memoryBudget',
|
||||
'analyze|--resource-budget <mb>': 'help.option.analyze.resourceBudget',
|
||||
'analyze|--workers <n>': 'help.option.analyze.workers',
|
||||
'analyze|--max-processes <n>': 'help.option.analyze.maxProcesses',
|
||||
'analyze|--max-process-branching <n>': 'help.option.analyze.maxProcessBranching',
|
||||
|
|
|
|||
|
|
@ -311,6 +311,8 @@ export const en = {
|
|||
'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).',
|
||||
'help.option.analyze.memoryBudget':
|
||||
'Main-thread V8 heap size in MB for analyze (integer >= 200). Re-runs analyze with exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size pin; parse workers keep their own heap caps.',
|
||||
'help.option.analyze.resourceBudget':
|
||||
'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.',
|
||||
'help.option.analyze.workers':
|
||||
'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.',
|
||||
'help.option.analyze.maxProcesses':
|
||||
|
|
|
|||
|
|
@ -284,6 +284,8 @@ export const zhCN = {
|
|||
'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值(字节,整数 >= -1;默认:67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。',
|
||||
'help.option.analyze.memoryBudget':
|
||||
'analyze 主线程 V8 堆大小(MB,整数 >= 200)。以该堆大小重新运行 analyze,覆盖按 RAM/cgroup 自动计算的上限及任何 --max-old-space-size 设置;解析 worker 各自保留独立的堆上限。',
|
||||
'help.option.analyze.resourceBudget':
|
||||
'分析进程的资源总预算(MiB),包括原生数据库缓冲池、解析 worker、嵌入模型和监督进程。',
|
||||
'help.option.analyze.workers':
|
||||
'解析 worker 池大小(>=1)。默认:cores-1,最多 16,按仓库规模自适应。',
|
||||
'help.option.analyze.maxProcesses':
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { localizeCliHelp } from './help-i18n.js';
|
|||
import { t } from './i18n/index.js';
|
||||
import { writeCommandBanner } from './command-banner.js';
|
||||
import { runProcessCliUpdateNotice } from './update-notice.js';
|
||||
import { memoryAutopilotDisabled } from '../core/ingestion/utils/effective-ram.js';
|
||||
|
||||
const program = new Command();
|
||||
|
||||
|
|
@ -178,6 +179,10 @@ program
|
|||
'exactly this heap, overriding the RAM/cgroup auto-sizer and any --max-old-space-size ' +
|
||||
'pin; parse workers keep their own heap caps.',
|
||||
)
|
||||
.option(
|
||||
'--resource-budget <mb>',
|
||||
'Total analysis resource budget in MiB, including native pools, workers, model and supervisor.',
|
||||
)
|
||||
.option(
|
||||
'--workers <n>',
|
||||
'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.',
|
||||
|
|
@ -286,14 +291,48 @@ program
|
|||
process.env.GITNEXUS_EMBEDDING_DIMS = dimsEnvBaseline;
|
||||
}
|
||||
})
|
||||
.action(
|
||||
createAnalyzerLbugLazyAction(
|
||||
.action(async (inputPath, options) => {
|
||||
// The connected continuation only publishes the writer's prepared graph
|
||||
// and embeds it. Its analyzer receipt is never used: the graph writer has
|
||||
// already finalized provenance. Keep the native check, without repeating
|
||||
// the complete analyzer identity snapshot in this separate process.
|
||||
// An explicit heap budget without the budget-child marker always re-execs
|
||||
// in ensureHeap before runFullAnalysis. Its parent receipt is unused too;
|
||||
// the child still captures provenance before importing the analyzer.
|
||||
const budgetSupervisor =
|
||||
!options.watch &&
|
||||
options.memoryBudget !== undefined &&
|
||||
process.env.GITNEXUS_HEAP_LIMIT_SOURCE !== 'budget';
|
||||
// An unconfigured automatic-heap parent also always re-execs with the
|
||||
// admitted old/semi-space flags. Keep conservative capture for custom Node
|
||||
// flags, manual heaps, test overrides and every analyzing child.
|
||||
const autoSupervisor =
|
||||
!options.watch &&
|
||||
options.memoryBudget === undefined &&
|
||||
process.env.GITNEXUS_ANALYSIS_SUPERVISED !== '1' &&
|
||||
process.env.GITNEXUS_ANALYSIS_MANUAL_HEAP !== '1' &&
|
||||
!memoryAutopilotDisabled() &&
|
||||
process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB === undefined &&
|
||||
process.execArgv.length === 0 &&
|
||||
!(process.env.NODE_OPTIONS ?? '').trim();
|
||||
if (
|
||||
budgetSupervisor ||
|
||||
autoSupervisor ||
|
||||
(process.env.GITNEXUS_ANALYSIS_CONTINUATION === '1' && process.connected)
|
||||
) {
|
||||
await createLbugLazyAction(() => import('./analyze.js'), 'analyzeCommand')(
|
||||
inputPath,
|
||||
options,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await createAnalyzerLbugLazyAction(
|
||||
() => import('../core/analyzer-identity.js'),
|
||||
() => import('./analyze.js'),
|
||||
'analyzeOrWatchCommandWithRunnerIdentity',
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
)(inputPath, options);
|
||||
});
|
||||
|
||||
program
|
||||
.command('index [path...]')
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ interface InvocationResolver {
|
|||
// no argument.
|
||||
resolveInvocationMode: (
|
||||
probe?: (command: string, gitnexusWrapper?: boolean) => string | null,
|
||||
deps?: { readonly npmMajor?: number | null },
|
||||
) => InvocationMode;
|
||||
formatDocumentationDlxCommand: (
|
||||
gitnexusArgs: string,
|
||||
|
|
@ -91,8 +92,17 @@ export function getNpmMajorVersion(): number | null {
|
|||
* cannot run.
|
||||
*/
|
||||
export function warnIfNpm11NpxRisk(): void {
|
||||
if (resolveInvocationMode() !== 'npx') return;
|
||||
const major = getNpmMajorVersion();
|
||||
// The resolver and warning need the same version. Probe once, lazily: a
|
||||
// preferred global binary or forced alternative still needs no npm spawn.
|
||||
let cachedMajor: number | null | undefined;
|
||||
const deps = {
|
||||
get npmMajor(): number | null {
|
||||
if (cachedMajor === undefined) cachedMajor = getNpmMajorVersion();
|
||||
return cachedMajor;
|
||||
},
|
||||
};
|
||||
if (resolveInvocationMode(undefined, deps) !== 'npx') return;
|
||||
const major = deps.npmMajor;
|
||||
if (major === null || major < 11) return;
|
||||
process.stderr.write(
|
||||
`Warning: npm ${major}.x can crash while installing gitnexus via npx ` +
|
||||
|
|
|
|||
|
|
@ -39,7 +39,9 @@ import {
|
|||
analyzerRunnerIdentitiesEqual,
|
||||
resolveAnalyzerRunnerIdentity,
|
||||
} from '../core/analyzer-identity.js';
|
||||
import { getIndexIncompleteReasons } from '../core/index-freshness.js';
|
||||
import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js';
|
||||
import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js';
|
||||
import { readIndexCapabilities } from '../storage/index-capabilities.js';
|
||||
import { getFtsDisabledReason, FTS_DISABLED_MESSAGE } from '../core/search/fts-policy.js';
|
||||
import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js';
|
||||
import {
|
||||
|
|
@ -218,6 +220,9 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
runnerIdentity: meta.runnerIdentity ?? null,
|
||||
runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown',
|
||||
incompleteReasons,
|
||||
graphIncompleteReasons: getGraphIncompleteReasons(meta),
|
||||
embeddingProgress: await readEmbeddingJobProgress(storagePath, meta),
|
||||
capabilities: await readIndexCapabilities(storagePath, meta),
|
||||
contentRetention: meta.contentRetention ?? 'full',
|
||||
},
|
||||
current: sourceAvailable ? { commit: getCurrentCommit(entry.path) } : null,
|
||||
|
|
@ -233,6 +238,14 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
console.log(`Index storage: ${storagePath}`);
|
||||
console.log(`Indexed: ${new Date(meta.indexedAt).toLocaleString()}`);
|
||||
console.log(`Indexed commit: ${meta.lastCommit?.slice(0, 7)}`);
|
||||
if (incompleteReasons.length) {
|
||||
console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`);
|
||||
console.log(
|
||||
`Graph incomplete reasons: ${JSON.stringify(payload.index.graphIncompleteReasons)}`,
|
||||
);
|
||||
}
|
||||
if (payload.index.embeddingProgress)
|
||||
console.log(`Embedding progress: ${JSON.stringify(payload.index.embeddingProgress)}`);
|
||||
console.log(
|
||||
sourceAvailable
|
||||
? 'Status: registered index (use status without --repo for working-tree freshness)'
|
||||
|
|
@ -310,12 +323,15 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
// commit comparison below then says whether it needs a re-analyze. Legacy/
|
||||
// no-branch metas and detached HEAD also fall through to the flat index.
|
||||
let activeMeta = repo.meta;
|
||||
let activeMetaDirectory = repo.storagePath;
|
||||
let workspaceLagsBranch = false;
|
||||
if (currentBranch && repo.meta.branch && currentBranch !== repo.meta.branch) {
|
||||
const { metaPath } = getStoragePaths(repo.repoPath, currentBranch, repo.storagePath);
|
||||
const branchMeta = await loadMeta(path.dirname(metaPath));
|
||||
if (branchMeta) activeMeta = branchMeta;
|
||||
else workspaceLagsBranch = true;
|
||||
if (branchMeta) {
|
||||
activeMeta = branchMeta;
|
||||
activeMetaDirectory = path.dirname(metaPath);
|
||||
} else workspaceLagsBranch = true;
|
||||
}
|
||||
|
||||
const currentRunnerIdentity = resolveAnalyzerRunnerIdentity(import.meta.url);
|
||||
|
|
@ -324,10 +340,11 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
currentRunnerIdentity,
|
||||
);
|
||||
const incompleteReasons = getIndexIncompleteReasons(activeMeta);
|
||||
const graphIncompleteReasons = getGraphIncompleteReasons(activeMeta);
|
||||
const metadataIsCurrent =
|
||||
currentCommit === activeMeta.lastCommit &&
|
||||
runnerIdentityIsCurrent &&
|
||||
incompleteReasons.length === 0;
|
||||
graphIncompleteReasons.length === 0;
|
||||
|
||||
// A matching HEAD is not enough: `analyze` re-indexes changed content at the
|
||||
// same commit, so the files the index covers must still be compared against
|
||||
|
|
@ -353,7 +370,10 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
contentDrift.reason === 'no-file-hashes' &&
|
||||
!isWorkingTreeDirty(repo.repoPath));
|
||||
|
||||
const isUpToDate = metadataIsCurrent && contentIsCurrent;
|
||||
const graphIsCurrent = metadataIsCurrent && contentIsCurrent;
|
||||
const isUpToDate = graphIsCurrent && incompleteReasons.length === 0;
|
||||
const embeddingProgress = await readEmbeddingJobProgress(activeMetaDirectory, activeMeta);
|
||||
const capabilities = await readIndexCapabilities(activeMetaDirectory, activeMeta);
|
||||
// Shared sibling store (#3352): which graph this checkout reads, and any
|
||||
// pre-adoption index still sitting in <repo>/.gitnexus.
|
||||
const storeRoot = storeRootOfCheckoutSlot(repo.storagePath);
|
||||
|
|
@ -386,10 +406,13 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
index: {
|
||||
indexedAt: activeMeta.indexedAt,
|
||||
commit: activeMeta.lastCommit,
|
||||
...(activeMeta.capabilities ? { capabilities: activeMeta.capabilities } : {}),
|
||||
...(capabilities ? { capabilities } : {}),
|
||||
runnerIdentity: activeMeta.runnerIdentity ?? null,
|
||||
runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown',
|
||||
incompleteReasons,
|
||||
graphIncompleteReasons,
|
||||
graphStatus: graphIsCurrent ? 'up-to-date' : 'stale',
|
||||
embeddingProgress,
|
||||
},
|
||||
current: {
|
||||
commit: currentCommit,
|
||||
|
|
@ -452,6 +475,10 @@ export const statusCommand = async (options: StatusOptions = {}) => {
|
|||
);
|
||||
if (incompleteReasons.length > 0) {
|
||||
console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`);
|
||||
console.log(`Graph status: ${graphIsCurrent ? 'up-to-date' : 'stale'}`);
|
||||
}
|
||||
if (embeddingProgress) {
|
||||
console.log(`Embedding progress: ${JSON.stringify(embeddingProgress)}`);
|
||||
}
|
||||
console.log(`${t('status.currentRunnerIdentity')}: ${JSON.stringify(currentRunnerIdentity)}`);
|
||||
if (contentDrift?.kind === 'current') {
|
||||
|
|
|
|||
266
gitnexus/src/core/analysis-embedding-phase.ts
Normal file
266
gitnexus/src/core/analysis-embedding-phase.ts
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
/** Common bounded embedding phase. Callers retain the ordinary slot writer lock. */
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import type { AnalyzeCallbacks } from './run-analyze.js';
|
||||
import type { AnalysisResourceBudget } from './analysis-resource-budget.js';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import { loadMeta, saveMeta } from '../storage/repo-manager.js';
|
||||
import { SCHEMA_FINGERPRINT } from './lbug/schema.js';
|
||||
import { openLbugConnection, closeLbugConnection } from './lbug/lbug-config.js';
|
||||
import { drainAnalysisNativeResources } from './lbug/lbug-adapter.js';
|
||||
import { assertLbugPublicationReady } from './lbug/publication.js';
|
||||
import { exportLegacyVectors } from './lbug/legacy-vector-export.js';
|
||||
import {
|
||||
beginInPlaceAnalysis,
|
||||
prepareAnalysisPublication,
|
||||
publishPreparedAnalysis,
|
||||
readPreparedAnalysis,
|
||||
} from './analysis-publication.js';
|
||||
import { verifyPublishedAnalysisGraph } from './analysis-graph-verification.js';
|
||||
import { runPublishedEmbeddingJob } from './embeddings/embedding-job.js';
|
||||
import { drainEmbeddingSidecarForAnalysis } from './embeddings/embedding-sidecar-client.js';
|
||||
import { deriveEmbeddingCap } from './embedding-mode.js';
|
||||
import { embeddingProgressFromState, type EmbeddingJobProgress } from './analysis-progress.js';
|
||||
import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from './analyzer-identity.js';
|
||||
import { resolveGraphPath } from '../storage/shared-store.js';
|
||||
|
||||
export function canPromoteLegacyGraph(metadata: RepoMeta): boolean {
|
||||
return (
|
||||
!metadata.graphGeneration &&
|
||||
!metadata.incrementalInProgress &&
|
||||
!metadata.graphWriteCollapsed &&
|
||||
!metadata.embeddingCheckpoint?.recovery &&
|
||||
metadata.schemaFingerprint === SCHEMA_FINGERPRINT &&
|
||||
metadata.runnerIdentity?.schemaVersion === ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION &&
|
||||
metadata.parseCacheIdentity !== undefined &&
|
||||
metadata.fileHashes !== undefined
|
||||
);
|
||||
}
|
||||
|
||||
/** Adopt a healthy legacy graph without parsing, rewriting or copying its graph.
|
||||
* Only its vector table is exported, retaining historical provenance for the
|
||||
* complete-group validator. Interrupted staged graphs take analyze's existing
|
||||
* recovery path instead. */
|
||||
export async function promoteLegacyGraph(
|
||||
directory: string,
|
||||
graphPath: string,
|
||||
metadata: RepoMeta,
|
||||
resources: AnalysisResourceBudget,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RepoMeta> {
|
||||
signal?.throwIfAborted();
|
||||
if (!canPromoteLegacyGraph(metadata))
|
||||
throw new Error(
|
||||
'Legacy graph provenance requires analysis before embedding; run gitnexus analyze first.',
|
||||
);
|
||||
if (
|
||||
path.dirname(graphPath) !== path.resolve(directory) ||
|
||||
path.basename(graphPath) !== (metadata.graphPath ?? 'lbug')
|
||||
)
|
||||
throw new Error('Legacy graph promotion requires an owned private graph');
|
||||
await drainEmbeddingSidecarForAnalysis();
|
||||
await drainAnalysisNativeResources();
|
||||
signal?.throwIfAborted();
|
||||
await assertLbugPublicationReady(graphPath);
|
||||
const handle = await openLbugConnection(lbug, graphPath, {
|
||||
readOnly: true,
|
||||
bufferPoolBytes: resources.embedding.graphPoolBytes,
|
||||
threads: resources.embedding.nativeThreads,
|
||||
});
|
||||
let legacy = metadata.legacyVectorSource;
|
||||
try {
|
||||
if (!legacy) {
|
||||
const historical = metadata.embeddingCheckpoint;
|
||||
legacy = await exportLegacyVectors({
|
||||
metadataDirectory: directory,
|
||||
query: async (sql) => {
|
||||
const result = await handle.conn.query(sql);
|
||||
const cursors = Array.isArray(result) ? result : [result];
|
||||
try {
|
||||
if (cursors.length !== 1) throw new Error('Unexpected legacy migration result');
|
||||
return await cursors[0].getAll();
|
||||
} finally {
|
||||
for (const cursor of cursors) cursor.close();
|
||||
}
|
||||
},
|
||||
provenance: {
|
||||
schemaFingerprint: metadata.schemaFingerprint!,
|
||||
...(historical
|
||||
? {
|
||||
recordedIdentity: {
|
||||
provider: historical.provider,
|
||||
model: historical.model,
|
||||
dimensions: historical.dimensions,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
excluded: historical?.pendingNodeIds ?? [],
|
||||
bufferPoolBytes: resources.embedding.vectorPoolBytes,
|
||||
threads: resources.embedding.nativeThreads,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await closeLbugConnection(handle, { strict: true, databasePath: graphPath });
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
// Persist the independent paid source before retiring the monolithic name.
|
||||
const retained = { ...metadata, ...(legacy ? { legacyVectorSource: legacy } : {}) };
|
||||
// Canonical legacy paths are represented by absence in the publication
|
||||
// contract; shared-store privatization can explicitly record that same name.
|
||||
if (retained.graphPath === 'lbug') delete retained.graphPath;
|
||||
await saveMeta(directory, retained);
|
||||
const now = Date.now();
|
||||
const dirty = await beginInPlaceAnalysis(
|
||||
directory,
|
||||
{
|
||||
...retained,
|
||||
incrementalInProgress: {
|
||||
startedAt: now,
|
||||
updatedAt: now,
|
||||
phase: 'legacy-migration',
|
||||
toWriteCount: 0,
|
||||
},
|
||||
},
|
||||
(meta) => saveMeta(directory, meta),
|
||||
);
|
||||
const attemptId = dirty.incrementalInProgress?.attemptId;
|
||||
if (!attemptId) throw new Error('Legacy migration has no owned publication attempt');
|
||||
const content = createHash('sha256');
|
||||
for (const entry of Object.entries(metadata.fileHashes!).sort(([a], [b]) =>
|
||||
a < b ? -1 : a > b ? 1 : 0,
|
||||
))
|
||||
content.update(JSON.stringify(entry)).update('\0');
|
||||
await prepareAnalysisPublication(directory, graphPath, retained, content.digest('hex'), {
|
||||
mode: 'in-place',
|
||||
attemptId,
|
||||
});
|
||||
const published = await publishPreparedAnalysis(directory, {
|
||||
saveMetadata: (meta) => saveMeta(directory, meta),
|
||||
verify: (file, meta) =>
|
||||
verifyPublishedAnalysisGraph(file, meta, {
|
||||
bufferPoolBytes: resources.embedding.graphPoolBytes,
|
||||
threads: resources.embedding.nativeThreads,
|
||||
}),
|
||||
});
|
||||
if (!published) throw new Error('Legacy graph publication disappeared');
|
||||
return published;
|
||||
}
|
||||
|
||||
export interface AnalysisEmbeddingPhaseOptions {
|
||||
metadataDirectory: string;
|
||||
graphPath: string;
|
||||
resources: AnalysisResourceBudget;
|
||||
callbacks: AnalyzeCallbacks;
|
||||
nodeLimit?: number;
|
||||
signal?: AbortSignal;
|
||||
/** Reject a superseded handoff before any model or paid provider request. */
|
||||
expectedGeneration?: string;
|
||||
}
|
||||
|
||||
export async function runAnalysisEmbeddingPhase(
|
||||
options: AnalysisEmbeddingPhaseOptions,
|
||||
): Promise<RepoMeta> {
|
||||
const { metadataDirectory, resources, callbacks, signal } = options;
|
||||
signal?.throwIfAborted();
|
||||
let metadata = await loadMeta(metadataDirectory);
|
||||
if (await readPreparedAnalysis(metadataDirectory))
|
||||
throw new Error('A prepared graph awaits publication; run gitnexus analyze before embedding.');
|
||||
if (!metadata || metadata.incrementalInProgress || metadata.graphWriteCollapsed)
|
||||
throw new Error('The graph is incomplete; run gitnexus analyze before embedding.');
|
||||
if (!Number.isSafeInteger(metadata.stats?.nodes) || (metadata.stats?.nodes ?? -1) < 0)
|
||||
throw new Error('The graph node count is unverified; run gitnexus analyze before embedding.');
|
||||
if (options.expectedGeneration && metadata.graphGeneration?.id !== options.expectedGeneration)
|
||||
throw new Error('Graph generation changed before embedding; retry analysis.');
|
||||
const cap = deriveEmbeddingCap(
|
||||
metadata.stats?.nodes ?? 0,
|
||||
metadata.embeddingJob || metadata.embeddingCheckpoint ? 0 : options.nodeLimit,
|
||||
);
|
||||
if (cap.skipForCap) {
|
||||
callbacks.onLog?.(
|
||||
`Embeddings skipped: graph exceeds the ${cap.nodeLimit}-node cap; use --embeddings 0 to disable it.`,
|
||||
);
|
||||
await callbacks.onGraphPublished?.();
|
||||
return metadata;
|
||||
}
|
||||
const { isHttpMode } = await import('./embeddings/http-client.js');
|
||||
if (!isHttpMode() && resources.embedding.modelBytes === 0)
|
||||
throw new Error(
|
||||
'Local embedding work was not admitted in this resource budget; retry analysis.',
|
||||
);
|
||||
if (metadata.embeddingCheckpoint) {
|
||||
const { decideEmbeddingResume } = await import('./embedding-checkpoint.js');
|
||||
const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js');
|
||||
const identity = resolveEmbeddingIdentity();
|
||||
const checkpoint = metadata.embeddingCheckpoint;
|
||||
if (Object.hasOwn(checkpoint, 'recovery'))
|
||||
throw new Error('The checkpoint references staged embeddings; run gitnexus analyze first.');
|
||||
const decision = decideEmbeddingResume(checkpoint, identity);
|
||||
if (decision.action === 'abort') throw new Error(decision.error);
|
||||
if (
|
||||
checkpoint.provider !== identity.provider ||
|
||||
checkpoint.model !== identity.model ||
|
||||
checkpoint.dimensions !== identity.dimensions
|
||||
)
|
||||
throw new Error(
|
||||
'The embedding checkpoint has a different provider, model or width; run gitnexus analyze --embeddings --force.',
|
||||
);
|
||||
callbacks.onLog?.(decision.log);
|
||||
}
|
||||
if (!metadata.graphGeneration)
|
||||
metadata = await promoteLegacyGraph(
|
||||
metadataDirectory,
|
||||
options.graphPath,
|
||||
metadata,
|
||||
resources,
|
||||
signal,
|
||||
);
|
||||
const graphPath = resolveGraphPath(metadataDirectory, metadata);
|
||||
let progress: EmbeddingJobProgress = { phase: 'preparing' };
|
||||
callbacks.onProgress('graph-ready', 90, 'Graph available; preparing embeddings', {
|
||||
graphPhase: 'ready',
|
||||
embeddingProgress: progress,
|
||||
});
|
||||
try {
|
||||
await callbacks.onGraphPublished?.();
|
||||
const result = await runPublishedEmbeddingJob({
|
||||
metadataDirectory,
|
||||
graphPath,
|
||||
resources: resources.embedding,
|
||||
signal,
|
||||
config: { threads: resources.embedding.nativeThreads },
|
||||
log: callbacks.onLog,
|
||||
metadata: {
|
||||
loadMetadata: () => loadMeta(metadataDirectory),
|
||||
saveMetadata: (meta) => saveMeta(metadataDirectory, meta),
|
||||
},
|
||||
onProgress: (state) => {
|
||||
progress = embeddingProgressFromState(state);
|
||||
const completed = state.completedNodes + state.reusedNodes;
|
||||
const percent =
|
||||
state.phase === 'indexing' ? 99 : 90 + 8 * (completed / Math.max(1, state.totalNodes));
|
||||
callbacks.onProgress(
|
||||
'embeddings',
|
||||
percent,
|
||||
state.phase === 'indexing'
|
||||
? 'Indexing completed embeddings'
|
||||
: `Embedding ${completed}/${state.totalNodes}`,
|
||||
{ graphPhase: 'ready', embeddingProgress: progress },
|
||||
);
|
||||
},
|
||||
});
|
||||
if (result.phase !== 'ready')
|
||||
throw new Error(
|
||||
'Embedding work remains incomplete; the graph is available. Retry to resume durable progress.',
|
||||
);
|
||||
return result.metadata;
|
||||
} catch (error) {
|
||||
callbacks.onProgress('embeddings', 98, 'Embeddings incomplete; graph remains available', {
|
||||
graphPhase: 'ready',
|
||||
embeddingProgress: { ...progress, phase: 'failed' },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
83
gitnexus/src/core/analysis-graph-verification.ts
Normal file
83
gitnexus/src/core/analysis-graph-verification.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
/** Read-only verification after the graph writer has physically released its DB. */
|
||||
import lbug from '@ladybugdb/core';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_FINGERPRINT } from './lbug/schema.js';
|
||||
import { closeLbugConnection, openLbugConnection, estimateBufferPool } from './lbug/lbug-config.js';
|
||||
import { assertLbugPublicationReady } from './lbug/publication.js';
|
||||
import { getFtsIndexes } from './search/fts-schema.js';
|
||||
import { verifySearchFTSIndexes } from './search/fts-indexes.js';
|
||||
import { ExtensionManager } from './lbug/extension-loader.js';
|
||||
|
||||
function count(value: unknown): number {
|
||||
if (typeof value !== 'number' && typeof value !== 'bigint')
|
||||
throw new Error('Published graph count is unverified');
|
||||
const result = Number(value);
|
||||
if (!Number.isSafeInteger(result) || result < 0)
|
||||
throw new Error('Published graph count is invalid');
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function verifyPublishedAnalysisGraph(
|
||||
graphPath: string,
|
||||
metadata: RepoMeta,
|
||||
resources: { bufferPoolBytes: number; threads: number },
|
||||
): Promise<void> {
|
||||
if (
|
||||
metadata.incrementalInProgress ||
|
||||
metadata.graphWriteCollapsed ||
|
||||
metadata.schemaFingerprint !== SCHEMA_FINGERPRINT ||
|
||||
!metadata.graphGeneration
|
||||
)
|
||||
throw new Error('Prepared graph has no compatible, verified schema generation');
|
||||
const expectedNodes = count(metadata.stats?.nodes);
|
||||
const expectedEdges = count(metadata.stats?.edges);
|
||||
await assertLbugPublicationReady(graphPath);
|
||||
const handle = await openLbugConnection(lbug, graphPath, {
|
||||
...resources,
|
||||
bufferPoolBytes: Math.min(
|
||||
resources.bufferPoolBytes,
|
||||
estimateBufferPool(expectedNodes + expectedEdges),
|
||||
),
|
||||
readOnly: true,
|
||||
});
|
||||
const query = async (sql: string): Promise<Record<string, unknown>[]> => {
|
||||
const result = await handle.conn.query(sql);
|
||||
const cursors = Array.isArray(result) ? result : [result];
|
||||
try {
|
||||
if (cursors.length !== 1) throw new Error('Unexpected publication verification result');
|
||||
return await cursors[0].getAll();
|
||||
} finally {
|
||||
for (const cursor of cursors) cursor.close();
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Binding every structural table also rejects incomplete native catalogs.
|
||||
// Count metadata is bounded; no source text, graph rows or vectors enter JS.
|
||||
const nodes = await query(
|
||||
NODE_TABLES.map((table) => `MATCH (n:\`${table}\`) RETURN count(n) AS count`).join(
|
||||
' UNION ALL ',
|
||||
),
|
||||
);
|
||||
const edges = await query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS count`);
|
||||
if (
|
||||
nodes.length !== NODE_TABLES.length ||
|
||||
edges.length !== 1 ||
|
||||
nodes.reduce((total, row) => total + count(row.count), 0) !== expectedNodes ||
|
||||
count(edges[0].count) !== expectedEdges
|
||||
)
|
||||
throw new Error('Prepared graph counts differ after writer release; artifacts retained');
|
||||
if (metadata.capabilities?.fts.status === 'available') {
|
||||
if (
|
||||
!(await new ExtensionManager().ensure(query, 'fts', 'publication verification', {
|
||||
policy: 'load-only',
|
||||
}))
|
||||
)
|
||||
throw new Error('Prepared graph FTS extension could not be loaded; artifacts retained');
|
||||
const missing = await verifySearchFTSIndexes(query, getFtsIndexes(metadata.ftsProfile));
|
||||
if (missing.length)
|
||||
throw new Error(`Prepared graph search indexes are incomplete: ${missing.join(', ')}`);
|
||||
}
|
||||
} finally {
|
||||
await closeLbugConnection(handle, { strict: true, databasePath: graphPath });
|
||||
}
|
||||
}
|
||||
89
gitnexus/src/core/analysis-host-resources.ts
Normal file
89
gitnexus/src/core/analysis-host-resources.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/** Host inputs for the pure phase-admission policy. No native/model imports. */
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { effectiveRamBytes } from './ingestion/utils/effective-ram.js';
|
||||
import { granuleRatio } from './lbug/lbug-config.js';
|
||||
import {
|
||||
resolveAnalysisResourceBudget,
|
||||
UNBOUNDED_GRAPH_POOL_MESSAGE,
|
||||
type AnalysisResourceBudget,
|
||||
type AnalysisResourceRequest,
|
||||
} from './analysis-resource-budget.js';
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
|
||||
function positiveEnvironment(name: string): number | undefined {
|
||||
const raw = process.env[name];
|
||||
if (raw === undefined || raw.trim() === '') return undefined;
|
||||
const value = Number(raw);
|
||||
if (name === 'GITNEXUS_LBUG_BUFFER_POOL_SIZE' && value === 0)
|
||||
throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE);
|
||||
if (!Number.isSafeInteger(value) || value < 1)
|
||||
throw new Error(`${name} must be a positive safe integer for bounded analysis`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export interface HostAnalysisResourceOptions {
|
||||
totalMb?: number;
|
||||
mainHeapMb?: number;
|
||||
workers?: number;
|
||||
includeParsing?: boolean;
|
||||
/** Local model capacity is only reserved when this invocation may embed. */
|
||||
includeEmbeddings: boolean;
|
||||
supervisorBytes?: number;
|
||||
/** Share one host envelope among concurrent auto-sync analyses. */
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export function resolveHostAnalysisResources(
|
||||
options: HostAnalysisResourceOptions,
|
||||
): AnalysisResourceBudget {
|
||||
const concurrency = options.concurrency ?? 1;
|
||||
if (!Number.isSafeInteger(concurrency) || concurrency < 1)
|
||||
throw new Error('Analysis concurrency must be a positive safe integer');
|
||||
const http = Boolean(process.env.GITNEXUS_EMBEDDING_URL && process.env.GITNEXUS_EMBEDDING_MODEL);
|
||||
const effectiveMemoryBytes = Math.floor(effectiveRamBytes() / concurrency);
|
||||
const request: AnalysisResourceRequest = {
|
||||
effectiveMemoryBytes,
|
||||
availableCpus: Math.max(1, Math.floor(os.availableParallelism() / concurrency)),
|
||||
totalMb: options.totalMb,
|
||||
mainHeapMb: options.mainHeapMb,
|
||||
allowHeapOvercommit: options.mainHeapMb !== undefined && options.totalMb === undefined,
|
||||
workers: options.workers ?? positiveEnvironment('GITNEXUS_WORKER_POOL_SIZE'),
|
||||
workerHeapMb: positiveEnvironment('GITNEXUS_WORKER_HEAP_MB'),
|
||||
embeddingThreads: options.includeEmbeddings
|
||||
? positiveEnvironment('GITNEXUS_EMBEDDING_THREADS')
|
||||
: undefined,
|
||||
preferredWorkers: 16,
|
||||
includeParsing: options.includeParsing,
|
||||
graphPoolBytes: positiveEnvironment('GITNEXUS_LBUG_BUFFER_POOL_SIZE'),
|
||||
graphPoolGranuleRatio: granuleRatio(),
|
||||
modelMemoryMb: options.includeEmbeddings && !http ? 2048 : 0,
|
||||
supervisorBytes: options.supervisorBytes,
|
||||
};
|
||||
return resolveAnalysisResourceBudget(request);
|
||||
}
|
||||
|
||||
/** Charge the retained frontend once, including a floor for its IPC/runtime. */
|
||||
export function analysisSupervisorReservation(): number {
|
||||
return Math.max(192 * MIB, Math.ceil(process.memoryUsage().rss / MIB) * MIB);
|
||||
}
|
||||
|
||||
/** Cheap preflight only. Core still revalidates the exact slot under its lock. */
|
||||
export async function analysisMayEmbed(
|
||||
repoPath: string,
|
||||
options: { embeddings?: boolean; force?: boolean; dropEmbeddings?: boolean; branch?: string },
|
||||
): Promise<boolean> {
|
||||
if (options.embeddings) return true;
|
||||
if (options.dropEmbeddings) return false;
|
||||
const { getStoragePaths, loadMeta } = await import('../storage/repo-manager.js');
|
||||
const { resolveBranchPlacement } = await import('../storage/branch-index.js');
|
||||
const placement = await resolveBranchPlacement(repoPath, options.branch ?? null);
|
||||
const directory = path.dirname(getStoragePaths(repoPath, placement.branch).metaPath);
|
||||
const meta = await loadMeta(directory);
|
||||
return Boolean(
|
||||
meta?.embeddingJob ||
|
||||
meta?.embeddingCheckpoint ||
|
||||
(options.force && (meta?.stats?.embeddings ?? 0) > 0),
|
||||
);
|
||||
}
|
||||
258
gitnexus/src/core/analysis-parse-cache.ts
Normal file
258
gitnexus/src/core/analysis-parse-cache.ts
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
/** Parser cache publication shared by an in-process writer and its exit supervisor. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import {
|
||||
loadParseCache,
|
||||
persistParseCacheChunk,
|
||||
pruneCache,
|
||||
saveParseCache,
|
||||
type ParseCache,
|
||||
} from '../storage/parse-cache.js';
|
||||
import { mergeStagedDurableParsedFileStore } from '../storage/parsedfile-store.js';
|
||||
import {
|
||||
embeddingJobDirectory,
|
||||
openEmbeddingJobFile,
|
||||
syncEmbeddingJobDirectory,
|
||||
} from '../storage/embedding-job.js';
|
||||
import { withStoreLock } from '../storage/shared-store-lifecycle.js';
|
||||
import type { SharedStoreLayout } from '../storage/shared-store.js';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import { listStoreMetaRoots } from './shared-store-analyze.js';
|
||||
|
||||
interface CachePublicationOptions {
|
||||
storagePath: string;
|
||||
metadataDirectory: string;
|
||||
cacheRoot: string;
|
||||
/** The exact published graph's handoff; never scan generation history. */
|
||||
publishedGeneration?: string;
|
||||
sharedStore?: SharedStoreLayout;
|
||||
collectBranchKeys: (
|
||||
root: string,
|
||||
exclude: string,
|
||||
) => Promise<{ keys: Set<string>; complete: boolean }>;
|
||||
log: (message: string) => void;
|
||||
}
|
||||
|
||||
/** Called only after graph publication, under the ordinary index lock. */
|
||||
export async function publishAnalysisParseCache(
|
||||
cache: ParseCache,
|
||||
options: CachePublicationOptions,
|
||||
): Promise<void> {
|
||||
const { storagePath, metadataDirectory, cacheRoot, sharedStore, collectBranchKeys, log } =
|
||||
options;
|
||||
const publish = async () => {
|
||||
const listing = sharedStore
|
||||
? await listStoreMetaRoots(sharedStore)
|
||||
: { roots: [storagePath], complete: true };
|
||||
const siblingKeys = new Set<string>();
|
||||
let complete = listing.complete;
|
||||
for (const root of listing.roots) {
|
||||
const folded = await collectBranchKeys(root, metadataDirectory);
|
||||
for (const key of folded.keys) siblingKeys.add(key);
|
||||
if (!folded.complete) complete = false;
|
||||
}
|
||||
// Re-read inside the publication lock: another checkout can have saved
|
||||
// since parsing began. Metadata alone never admits an incompatible shard.
|
||||
const retained = await loadParseCache(
|
||||
cacheRoot,
|
||||
cache.identity,
|
||||
cache.writeStoragePath ?? cacheRoot,
|
||||
);
|
||||
cache.storagePath = cacheRoot;
|
||||
cache.onDiskKeys = new Set();
|
||||
for (const key of retained.onDiskKeys ?? []) {
|
||||
if (
|
||||
!cache.stagedKeys?.has(key) &&
|
||||
!cache.staleKeys?.has(key) &&
|
||||
(cache.usedKeys.has(key) || !complete || siblingKeys.has(key))
|
||||
)
|
||||
cache.onDiskKeys.add(key);
|
||||
}
|
||||
for (const [key, value] of retained.entries) {
|
||||
if (
|
||||
!cache.usedKeys.has(key) &&
|
||||
!cache.stagedKeys?.has(key) &&
|
||||
!cache.staleKeys?.has(key) &&
|
||||
(!complete || siblingKeys.has(key))
|
||||
)
|
||||
cache.entries.set(key, value);
|
||||
}
|
||||
if (complete) {
|
||||
for (const key of siblingKeys) cache.usedKeys.add(key);
|
||||
} else {
|
||||
log('Parse cache: a branch meta was unreadable — retaining all cached chunks (#2106).');
|
||||
for (const key of cache.entries.keys()) cache.usedKeys.add(key);
|
||||
for (const key of cache.onDiskKeys) cache.usedKeys.add(key);
|
||||
}
|
||||
const pruned = pruneCache(cache, cache.usedKeys);
|
||||
if (pruned > 0) log(`Parse cache: pruned ${pruned} stale chunk entries`);
|
||||
const savedKeys = await saveParseCache(cacheRoot, cache);
|
||||
await mergeStagedDurableParsedFileStore(
|
||||
cacheRoot,
|
||||
cache.writeStoragePath ?? cacheRoot,
|
||||
cache.identity.base,
|
||||
new Set(savedKeys),
|
||||
);
|
||||
};
|
||||
await (sharedStore ? withStoreLock(sharedStore, 'cache', publish) : publish());
|
||||
if (options.publishedGeneration) {
|
||||
const root = await embeddingJobDirectory(metadataDirectory);
|
||||
await fs.rm(path.join(root, receiptName(options.publishedGeneration)), { force: true });
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
}
|
||||
}
|
||||
|
||||
// This contains only chunk keys, never ASTs, graph data or source content.
|
||||
// The fixed ceiling fits the phase headroom; failure merely leaves this
|
||||
// optional cache unpublished, and never changes graph/embedding correctness.
|
||||
const MAX_HANDOFF_BYTES = 16 * 1024 * 1024;
|
||||
const KEY = /^[a-f0-9]{64}$/;
|
||||
interface CacheHandoff {
|
||||
version: 1;
|
||||
generation: string;
|
||||
stage: string;
|
||||
identity: ParseCache['identity'];
|
||||
used: string[];
|
||||
staged: string[];
|
||||
stale: string[];
|
||||
}
|
||||
const checksum = (payload: string) => createHash('sha256').update(payload).digest('hex');
|
||||
const receiptName = (generation: string) => {
|
||||
if (!/^[a-f0-9]{32}$/.test(generation)) throw new Error('Invalid cache publication generation');
|
||||
return `analysis-cache-${generation}.json`;
|
||||
};
|
||||
|
||||
async function ownedStage(root: string, relative: string): Promise<string> {
|
||||
if (!/^parse-rebuild\/run\.[A-Za-z0-9]{6}$/.test(relative))
|
||||
throw new Error('Invalid prepared parser cache stage');
|
||||
let current = await embeddingJobDirectory(root);
|
||||
for (const part of relative.split('/'))
|
||||
current = await embeddingJobDirectory(path.join(current, part));
|
||||
return current;
|
||||
}
|
||||
|
||||
/** Flush only memory fallback shards. Existing live shards stay in place;
|
||||
* the supervisor will re-read their current identity under the cache lock. */
|
||||
export async function stageAnalysisParseCache(
|
||||
root: string,
|
||||
generation: string,
|
||||
cache: ParseCache,
|
||||
): Promise<void> {
|
||||
root = await embeddingJobDirectory(root);
|
||||
const name = receiptName(generation);
|
||||
if (!cache.writeStoragePath)
|
||||
throw new Error('Prepared parser cache has no owned staging directory');
|
||||
const relative = path.relative(root, cache.writeStoragePath).split(path.sep).join('/');
|
||||
await ownedStage(root, relative);
|
||||
// Bound each selection before copying or serializing it. Unknown/malformed
|
||||
// keys cannot become filesystem paths.
|
||||
for (const keys of [cache.usedKeys, cache.stagedKeys, cache.staleKeys]) {
|
||||
if (!keys) continue;
|
||||
if (keys.size > Math.floor(MAX_HANDOFF_BYTES / 3 / 70))
|
||||
throw new Error('Prepared parser cache key set exceeds its bound');
|
||||
for (const key of keys)
|
||||
if (!KEY.test(key)) throw new Error('Prepared parser cache contains an invalid key');
|
||||
}
|
||||
for (const [key, value] of cache.entries) await persistParseCacheChunk(cache, key, value);
|
||||
if (cache.entries.size)
|
||||
throw new Error('Could not persist prepared parser cache fallback shards');
|
||||
const payload: CacheHandoff = {
|
||||
version: 1,
|
||||
generation,
|
||||
stage: relative,
|
||||
identity: cache.identity,
|
||||
used: [...cache.usedKeys],
|
||||
staged: [...(cache.stagedKeys ?? [])],
|
||||
stale: [...(cache.staleKeys ?? [])],
|
||||
};
|
||||
const serialized = JSON.stringify(payload);
|
||||
const bytes = Buffer.from(JSON.stringify({ payload, checksum: checksum(serialized) }));
|
||||
if (bytes.length > MAX_HANDOFF_BYTES)
|
||||
throw new Error('Prepared parser cache receipt exceeds its bound');
|
||||
const temporary = path.join(root, `.${name}.${randomUUID()}.tmp`);
|
||||
const handle = await fs.open(temporary, 'wx', 0o600);
|
||||
try {
|
||||
try {
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await fs.rename(temporary, path.join(root, name));
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
} finally {
|
||||
await fs.rm(temporary, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** The published graph binds the generation, parser identity and used keys.
|
||||
* Missing cache state is optional; malformed state is never used as a cache. */
|
||||
export async function loadAnalysisParseCache(
|
||||
root: string,
|
||||
metadata: RepoMeta,
|
||||
): Promise<ParseCache | undefined> {
|
||||
const generation = metadata.graphGeneration?.id;
|
||||
if (!generation) throw new Error('Cache publication requires a published graph generation');
|
||||
root = await embeddingJobDirectory(root);
|
||||
const file = path.join(root, receiptName(generation));
|
||||
let opened;
|
||||
try {
|
||||
opened = await openEmbeddingJobFile(file);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
throw error;
|
||||
}
|
||||
const { handle, stat } = opened;
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
if (stat.size > MAX_HANDOFF_BYTES) throw new Error('Unverified prepared parser cache receipt');
|
||||
bytes = Buffer.alloc(stat.size + 1);
|
||||
let length = 0;
|
||||
while (length < bytes.length) {
|
||||
const result = await handle.read(bytes, length, bytes.length - length, length);
|
||||
if (!result.bytesRead) break;
|
||||
length += result.bytesRead;
|
||||
}
|
||||
if (length !== stat.size) throw new Error('Prepared parser cache receipt changed');
|
||||
bytes = bytes.subarray(0, length);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
const envelope = JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(bytes));
|
||||
const payload = envelope.payload as CacheHandoff;
|
||||
const keys = (value: unknown): value is string[] =>
|
||||
Array.isArray(value) && value.every((key) => typeof key === 'string' && KEY.test(key));
|
||||
if (
|
||||
!payload ||
|
||||
envelope.checksum !== checksum(JSON.stringify(payload)) ||
|
||||
payload.version !== 1 ||
|
||||
payload.generation !== generation ||
|
||||
typeof payload.stage !== 'string' ||
|
||||
!payload.identity ||
|
||||
typeof payload.identity.base !== 'string' ||
|
||||
typeof payload.identity.pdg !== 'string' ||
|
||||
payload.identity.base !== metadata.parseCacheIdentity?.base ||
|
||||
(metadata.parseCacheIdentity.pdg &&
|
||||
payload.identity.pdg !== metadata.parseCacheIdentity.pdg.identity) ||
|
||||
!keys(payload.used) ||
|
||||
!keys(payload.staged) ||
|
||||
!keys(payload.stale)
|
||||
)
|
||||
throw new Error('Prepared parser cache does not match the published graph');
|
||||
const usedKeys = new Set(metadata.cacheKeys ?? []);
|
||||
if (
|
||||
payload.used.length !== usedKeys.size ||
|
||||
new Set(payload.used).size !== usedKeys.size ||
|
||||
payload.used.some((key) => !usedKeys.has(key))
|
||||
)
|
||||
throw new Error('Prepared parser cache selection differs from the published graph');
|
||||
return {
|
||||
identity: payload.identity,
|
||||
entries: new Map(),
|
||||
usedKeys,
|
||||
writeStoragePath: await ownedStage(root, payload.stage),
|
||||
stagedKeys: new Set(payload.staged),
|
||||
staleKeys: new Set(payload.stale),
|
||||
};
|
||||
}
|
||||
156
gitnexus/src/core/analysis-phase-supervisor.ts
Normal file
156
gitnexus/src/core/analysis-phase-supervisor.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
/** Filesystem-only handoff from an exit-only graph writer. No model/native imports. */
|
||||
import path from 'node:path';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import { requireExclusiveIndexLock } from '../storage/index-lock.js';
|
||||
import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js';
|
||||
import { embeddingJobDirectory } from '../storage/embedding-job.js';
|
||||
import { loadMeta } from '../storage/repo-meta.js';
|
||||
import {
|
||||
publishPreparedAnalysis,
|
||||
readPreparedAnalysis,
|
||||
type AnalysisPublicationActions,
|
||||
} from './analysis-publication.js';
|
||||
|
||||
export interface AnalysisPhaseTarget {
|
||||
repoPath: string;
|
||||
storagePath: string;
|
||||
metadataDirectory: string;
|
||||
repoName: string;
|
||||
commit: string;
|
||||
}
|
||||
|
||||
/** Only this bounded receipt crosses IPC; the live graph stays in its child. */
|
||||
export interface PreparedAnalysisHandoff extends AnalysisPhaseTarget {
|
||||
version: 1;
|
||||
publicationId: string;
|
||||
}
|
||||
|
||||
export function projectPreparedAnalysisHandoff(value: unknown): PreparedAnalysisHandoff {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
throw new Error('Invalid prepared analysis handoff');
|
||||
const input = value as Record<string, unknown>;
|
||||
const text = (name: string, maximum: number, allowEmpty = false): string => {
|
||||
const value = input[name];
|
||||
if (typeof value !== 'string' || (!allowEmpty && !value.length) || value.length > maximum)
|
||||
throw new Error(`Invalid prepared analysis ${name}`);
|
||||
return value;
|
||||
};
|
||||
const projected: PreparedAnalysisHandoff = {
|
||||
version: 1,
|
||||
publicationId: text('publicationId', 32),
|
||||
repoPath: text('repoPath', 32768),
|
||||
storagePath: text('storagePath', 32768),
|
||||
metadataDirectory: text('metadataDirectory', 32768),
|
||||
repoName: text('repoName', 1024),
|
||||
commit: text('commit', 64, true),
|
||||
};
|
||||
if (
|
||||
input.version !== 1 ||
|
||||
!/^[a-f0-9]{32}$/.test(projected.publicationId) ||
|
||||
!/^(?:[a-f0-9]{40}|[a-f0-9]{64})?$/.test(projected.commit) ||
|
||||
![projected.repoPath, projected.storagePath, projected.metadataDirectory].every(path.isAbsolute)
|
||||
)
|
||||
throw new Error('Invalid prepared analysis identity or location');
|
||||
return projected;
|
||||
}
|
||||
|
||||
export interface AnalysisPhasePublicationActions extends AnalysisPublicationActions {
|
||||
/** Re-resolve checkout placement, current HEAD and the registered name. */
|
||||
resolveTarget: () => Promise<AnalysisPhaseTarget>;
|
||||
/** Finish registry/shared-store/cache bookkeeping and reader refresh. Called
|
||||
* under the ordinary slot lock after native verification and publication.
|
||||
* A cancellation after publication must still leave this graph available. */
|
||||
finalizeGraph: (metadata: RepoMeta) => Promise<RepoMeta>;
|
||||
signal?: AbortSignal;
|
||||
log?: (message: string) => void;
|
||||
onWaitStart?: () => void;
|
||||
}
|
||||
|
||||
function requireSameTarget(expected: AnalysisPhaseTarget, actual: AnalysisPhaseTarget): void {
|
||||
if (
|
||||
expected.repoName !== actual.repoName ||
|
||||
expected.commit !== actual.commit ||
|
||||
(['repoPath', 'storagePath', 'metadataDirectory'] as const).some(
|
||||
(key) => path.resolve(expected[key]) !== path.resolve(actual[key]),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
'Analysis slot, checkout or registered name changed during the graph handoff; retry before embedding',
|
||||
);
|
||||
}
|
||||
|
||||
/** The launcher calls this only after its actual-exit promise resolves. The
|
||||
* persisted writer-PID guard independently rejects an early or stale handoff.
|
||||
* No borrowed lock: acquire normally, re-resolve under ownership, then finish
|
||||
* the exact prepared attempt. An embedding continuation must acquire its own
|
||||
* lock and recheck this returned generation before making paid requests. */
|
||||
export async function publishAnalysisAfterProcessExit(
|
||||
handoff: PreparedAnalysisHandoff,
|
||||
writerPid: number,
|
||||
actions: AnalysisPhasePublicationActions,
|
||||
): Promise<RepoMeta> {
|
||||
const expected = projectPreparedAnalysisHandoff(handoff);
|
||||
if (!Number.isSafeInteger(writerPid) || writerPid < 1)
|
||||
throw new Error('Missing graph writer PID');
|
||||
actions.signal?.throwIfAborted();
|
||||
requireSameTarget(expected, await actions.resolveTarget());
|
||||
const root = await embeddingJobDirectory(expected.metadataDirectory);
|
||||
const lock = await acquireAnalysisSlotLock(root, {
|
||||
sweep: false,
|
||||
log: actions.log,
|
||||
onWaitStart: actions.onWaitStart,
|
||||
});
|
||||
try {
|
||||
requireExclusiveIndexLock(
|
||||
lock,
|
||||
'Graph publication requires exclusive ownership of its index slot',
|
||||
);
|
||||
actions.signal?.throwIfAborted();
|
||||
requireSameTarget(expected, await actions.resolveTarget());
|
||||
const prepared = await readPreparedAnalysis(root);
|
||||
if (prepared && (prepared.id !== expected.publicationId || prepared.writerPid !== writerPid))
|
||||
throw new Error(
|
||||
'Prepared graph attempt changed during handoff; artifacts retained; retry analysis',
|
||||
);
|
||||
const metadata = prepared
|
||||
? await publishPreparedAnalysis(root, {
|
||||
saveMetadata: actions.saveMetadata,
|
||||
verify: async (file, metadata) => {
|
||||
if (metadata.repoPath !== expected.repoPath || metadata.lastCommit !== expected.commit)
|
||||
throw new Error('Prepared graph belongs to a different checkout');
|
||||
actions.signal?.throwIfAborted();
|
||||
await actions.verify(file, metadata);
|
||||
actions.signal?.throwIfAborted();
|
||||
},
|
||||
})
|
||||
: await loadMeta(root);
|
||||
// Another waiting writer may recover this exact receipt after the graph
|
||||
// child exits. Adopt only the clean, already-published generation.
|
||||
if (
|
||||
!metadata ||
|
||||
metadata.graphGeneration?.id !== expected.publicationId ||
|
||||
metadata.repoPath !== expected.repoPath ||
|
||||
metadata.lastCommit !== expected.commit ||
|
||||
metadata.incrementalInProgress ||
|
||||
metadata.graphWriteCollapsed
|
||||
)
|
||||
throw new Error(
|
||||
'Prepared graph attempt changed during handoff; artifacts retained; retry analysis',
|
||||
);
|
||||
// Once publication starts, complete discoverability even if cancellation
|
||||
// arrived during its filesystem operations. Paid work remains inadmissible.
|
||||
const finalized = await actions.finalizeGraph(metadata);
|
||||
requireSameTarget(expected, await actions.resolveTarget());
|
||||
if (
|
||||
finalized.graphGeneration?.id !== expected.publicationId ||
|
||||
finalized.repoPath !== expected.repoPath ||
|
||||
finalized.incrementalInProgress ||
|
||||
finalized.graphWriteCollapsed
|
||||
)
|
||||
throw new Error('Graph finalization did not retain the published generation');
|
||||
actions.signal?.throwIfAborted();
|
||||
return finalized;
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
156
gitnexus/src/core/analysis-progress.ts
Normal file
156
gitnexus/src/core/analysis-progress.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import type { EmbeddingJobState } from '../storage/embedding-job-journal.js';
|
||||
|
||||
export type GraphAnalysisPhase = 'pending' | 'building' | 'ready' | 'failed';
|
||||
|
||||
/** Counts describe durable acknowledgements. Active work is part of pending,
|
||||
* never an additional category. Totals remain absent before manifest preflight. */
|
||||
export interface EmbeddingJobProgress {
|
||||
phase: 'preparing' | 'embedding' | 'indexing' | 'ready' | 'preserved' | 'failed' | 'unverified';
|
||||
totalUnits?: number;
|
||||
completedUnits?: number;
|
||||
reusedUnits?: number;
|
||||
pendingUnits?: number;
|
||||
failedUnits?: number;
|
||||
activeUnit?: number | null;
|
||||
totalNodes?: number;
|
||||
completedNodes?: number;
|
||||
reusedNodes?: number;
|
||||
pendingNodes?: number;
|
||||
failedNodes?: number;
|
||||
excludedNodes?: number;
|
||||
chunks?: number;
|
||||
}
|
||||
|
||||
/** Additive fields shared by CLI callbacks, worker IPC, and HTTP progress. */
|
||||
export interface AnalysisPhaseProgress {
|
||||
graphPhase?: GraphAnalysisPhase;
|
||||
embeddingProgress?: EmbeddingJobProgress;
|
||||
modelDownloadPercent?: number;
|
||||
}
|
||||
|
||||
const publicEmbeddingProgress = (
|
||||
progress: EmbeddingJobProgress | undefined,
|
||||
): EmbeddingJobProgress | undefined => {
|
||||
if (
|
||||
!progress ||
|
||||
!['preparing', 'embedding', 'indexing', 'ready', 'preserved', 'failed', 'unverified'].includes(
|
||||
progress.phase,
|
||||
)
|
||||
)
|
||||
return undefined;
|
||||
const safe: EmbeddingJobProgress = { phase: progress.phase };
|
||||
// Explicit numeric allowlist: the source may also contain internal paths,
|
||||
// identities, credentials, or values that cannot cross JSON IPC.
|
||||
for (const key of [
|
||||
'totalUnits',
|
||||
'completedUnits',
|
||||
'reusedUnits',
|
||||
'pendingUnits',
|
||||
'failedUnits',
|
||||
'totalNodes',
|
||||
'completedNodes',
|
||||
'reusedNodes',
|
||||
'pendingNodes',
|
||||
'failedNodes',
|
||||
'excludedNodes',
|
||||
'chunks',
|
||||
] as const) {
|
||||
const value = progress[key];
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) safe[key] = value;
|
||||
}
|
||||
if (
|
||||
progress.activeUnit === null ||
|
||||
(typeof progress.activeUnit === 'number' &&
|
||||
Number.isSafeInteger(progress.activeUnit) &&
|
||||
progress.activeUnit >= 0)
|
||||
)
|
||||
safe.activeUnit = progress.activeUnit;
|
||||
return safe;
|
||||
};
|
||||
|
||||
/** The same allowlist protects worker IPC and public progress feeds. */
|
||||
export function projectAnalysisPhaseProgress(
|
||||
progress: AnalysisPhaseProgress | undefined,
|
||||
): AnalysisPhaseProgress {
|
||||
const result: AnalysisPhaseProgress = {};
|
||||
if (!progress) return result;
|
||||
if (
|
||||
progress.graphPhase &&
|
||||
['pending', 'building', 'ready', 'failed'].includes(progress.graphPhase)
|
||||
)
|
||||
result.graphPhase = progress.graphPhase;
|
||||
const embeddingProgress = publicEmbeddingProgress(progress.embeddingProgress);
|
||||
if (embeddingProgress) result.embeddingProgress = embeddingProgress;
|
||||
if (
|
||||
typeof progress.modelDownloadPercent === 'number' &&
|
||||
Number.isFinite(progress.modelDownloadPercent) &&
|
||||
progress.modelDownloadPercent >= 0 &&
|
||||
progress.modelDownloadPercent <= 100
|
||||
)
|
||||
result.modelDownloadPercent = progress.modelDownloadPercent;
|
||||
return result;
|
||||
}
|
||||
|
||||
export function embeddingProgressFromState(state: EmbeddingJobState): EmbeddingJobProgress {
|
||||
const counts = [
|
||||
state.totalUnits,
|
||||
state.completedUnits,
|
||||
state.reusedUnits,
|
||||
state.failedUnits,
|
||||
state.totalNodes,
|
||||
state.completedNodes,
|
||||
state.reusedNodes,
|
||||
state.failedNodes,
|
||||
state.excludedNodes ?? 0,
|
||||
state.chunks,
|
||||
];
|
||||
const pendingUnits =
|
||||
state.totalUnits - state.completedUnits - state.reusedUnits - state.failedUnits;
|
||||
const pendingNodes =
|
||||
state.totalNodes -
|
||||
state.completedNodes -
|
||||
state.reusedNodes -
|
||||
state.failedNodes -
|
||||
(state.excludedNodes ?? 0);
|
||||
const activeUnit = state.active?.start?.unit ?? null;
|
||||
if (
|
||||
counts.some((n) => !Number.isSafeInteger(n) || n < 0) ||
|
||||
pendingUnits < 0 ||
|
||||
pendingNodes < 0 ||
|
||||
state.active === undefined ||
|
||||
!['embedding', 'indexing', 'ready'].includes(state.phase) ||
|
||||
(state.active !== null &&
|
||||
(!Number.isSafeInteger(activeUnit) ||
|
||||
activeUnit === null ||
|
||||
activeUnit < 0 ||
|
||||
activeUnit >= state.totalUnits ||
|
||||
pendingUnits < 1 ||
|
||||
!Number.isSafeInteger(state.active.nodes) ||
|
||||
state.active.nodes < 1 ||
|
||||
state.active.nodes > pendingNodes)) ||
|
||||
(state.phase !== 'embedding' &&
|
||||
(pendingUnits ||
|
||||
pendingNodes ||
|
||||
state.failedUnits ||
|
||||
state.failedNodes ||
|
||||
state.active !== null))
|
||||
) {
|
||||
throw new Error('Embedding progress accounting is unverified');
|
||||
}
|
||||
return {
|
||||
phase: state.phase,
|
||||
totalUnits: state.totalUnits,
|
||||
completedUnits: state.completedUnits,
|
||||
reusedUnits: state.reusedUnits,
|
||||
pendingUnits,
|
||||
failedUnits: state.failedUnits,
|
||||
activeUnit,
|
||||
totalNodes: state.totalNodes,
|
||||
completedNodes: state.completedNodes,
|
||||
reusedNodes: state.reusedNodes,
|
||||
pendingNodes,
|
||||
failedNodes: state.failedNodes,
|
||||
...(state.excludedNodes ? { excludedNodes: state.excludedNodes } : {}),
|
||||
chunks: state.chunks,
|
||||
};
|
||||
}
|
||||
487
gitnexus/src/core/analysis-publication.ts
Normal file
487
gitnexus/src/core/analysis-publication.ts
Normal file
|
|
@ -0,0 +1,487 @@
|
|||
/** Durable graph publication. Native verification/close and metadata ownership
|
||||
* are supplied by the coordinator, which holds the index-directory write lock. */
|
||||
import fs from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { RepoMeta } from '../storage/repo-meta.js';
|
||||
import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from '../storage/storage-constants.js';
|
||||
import { preserveVectorCoverage } from '../storage/vector-generation.js';
|
||||
import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../storage/embedding-job.js';
|
||||
import { readProcStartTime } from '../storage/index-lock.js';
|
||||
import { reclaimAnalysisArtifacts } from '../storage/analysis-artifact-cleanup.js';
|
||||
|
||||
const RECEIPT = 'analysis-publication.json';
|
||||
const GENERATION = /^[a-f0-9]{32}$/;
|
||||
const PRIVATE_GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/;
|
||||
const DIGEST = /^[a-f0-9]{64}$/;
|
||||
const STAGING = /^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/;
|
||||
const MAX_RECEIPT_BYTES = 4096;
|
||||
const MAX_METADATA_BYTES = 256 * 1024 * 1024;
|
||||
const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex');
|
||||
// Share the incarnation across loaders of this module within one process.
|
||||
const INSTANCE_KEY = Symbol.for('gitnexus.analysis-publication.writer-instance');
|
||||
const processIdentity = process as unknown as Record<symbol, string>;
|
||||
const WRITER_INSTANCE = (processIdentity[INSTANCE_KEY] ??= randomUUID().replaceAll('-', ''));
|
||||
|
||||
export interface PreparedAnalysisPublication {
|
||||
version: 2;
|
||||
mode: 'staged' | 'in-place';
|
||||
id: string;
|
||||
/** Source basename; also names the existing file for in-place writes. */
|
||||
stagingFile: string;
|
||||
graphFile: string;
|
||||
metadataFile: string;
|
||||
metadataBytes: number;
|
||||
metadataDigest: string;
|
||||
/** Exact prior metadata, or absence, observed under the slot's write lock. */
|
||||
priorMetadataDigest: string | null;
|
||||
/** Permitted intermediate state while retiring a canonical legacy graph. */
|
||||
redirectedMetadataDigest?: string;
|
||||
/** Exit-only callers may not publish while this process is still alive. */
|
||||
writerPid?: number;
|
||||
writerStartTime?: string;
|
||||
writerInstance?: string;
|
||||
/** A hard-link destination for the previous legacy graph; never copied. */
|
||||
legacyGraphFile?: string;
|
||||
}
|
||||
|
||||
export type AnalysisPublicationOptions = (
|
||||
| { mode?: 'staged' }
|
||||
| { mode: 'in-place'; attemptId: string }
|
||||
) & { waitForProcessExit?: boolean };
|
||||
|
||||
export interface AnalysisPublicationActions {
|
||||
/** Open read-only and verify graph/schema/FTS against the prepared metadata.
|
||||
* Must close the verification handle before returning (including on Windows).
|
||||
* Unknown counts/catalog state must reject, never certify an empty graph.
|
||||
*/
|
||||
verify: (graphPath: string, metadata: RepoMeta) => Promise<void>;
|
||||
/** The normal metadata writer, including its compatibility mirror. */
|
||||
saveMetadata: (metadata: RepoMeta) => Promise<void>;
|
||||
}
|
||||
|
||||
function validateReceipt(value: unknown): asserts value is PreparedAnalysisPublication {
|
||||
const r = value as PreparedAnalysisPublication;
|
||||
if (
|
||||
!r ||
|
||||
r.version !== 2 ||
|
||||
!GENERATION.test(r.id) ||
|
||||
!PRIVATE_GRAPH.test(r.graphFile) ||
|
||||
(r.mode === 'staged'
|
||||
? !STAGING.test(r.stagingFile) || r.graphFile !== `lbug.generation.${r.id}`
|
||||
: r.mode !== 'in-place' ||
|
||||
(r.stagingFile === 'lbug'
|
||||
? r.graphFile !== `lbug.generation.${r.id}`
|
||||
: !PRIVATE_GRAPH.test(r.stagingFile) || r.graphFile !== r.stagingFile)) ||
|
||||
r.metadataFile !== `analysis-meta-${r.id}.json` ||
|
||||
(r.priorMetadataDigest !== null && !DIGEST.test(r.priorMetadataDigest)) ||
|
||||
(r.redirectedMetadataDigest !== undefined &&
|
||||
(!r.legacyGraphFile || !DIGEST.test(r.redirectedMetadataDigest))) ||
|
||||
(r.writerPid !== undefined && (!Number.isSafeInteger(r.writerPid) || r.writerPid < 1)) ||
|
||||
(r.writerStartTime !== undefined && (!r.writerPid || !/^\d{1,32}$/.test(r.writerStartTime))) ||
|
||||
(r.writerInstance !== undefined && (!r.writerPid || !GENERATION.test(r.writerInstance))) ||
|
||||
(r.legacyGraphFile !== undefined &&
|
||||
(r.mode !== 'staged' ||
|
||||
!PRIVATE_GRAPH.test(r.legacyGraphFile) ||
|
||||
r.legacyGraphFile === r.graphFile)) ||
|
||||
!Number.isSafeInteger(r.metadataBytes) ||
|
||||
r.metadataBytes < 1 ||
|
||||
r.metadataBytes > MAX_METADATA_BYTES ||
|
||||
!DIGEST.test(r.metadataDigest)
|
||||
)
|
||||
throw new Error('Invalid prepared graph publication; artifacts retained');
|
||||
}
|
||||
|
||||
async function regularFile(file: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(file);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Publication artifact must be an owned regular file');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function boundedRead(file: string, maximum: number): Promise<Buffer> {
|
||||
const handle = await fs.open(
|
||||
file,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0),
|
||||
);
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
const entry = await fs.lstat(file);
|
||||
if (!entry.isFile() || entry.isSymbolicLink())
|
||||
throw new Error('Publication artifact must be an owned regular file');
|
||||
if (stat.dev !== entry.dev || stat.ino !== entry.ino)
|
||||
throw new Error('Publication artifact changed while opening');
|
||||
if (!stat.isFile() || stat.size > maximum)
|
||||
throw new Error('Publication artifact exceeds its receipt bound');
|
||||
const buffer = Buffer.alloc(stat.size + 1);
|
||||
let size = 0;
|
||||
while (size < buffer.length) {
|
||||
const result = await handle.read(buffer, size, buffer.length - size, size);
|
||||
if (!result.bytesRead) break;
|
||||
size += result.bytesRead;
|
||||
}
|
||||
if (size !== stat.size) throw new Error('Publication artifact changed while reading');
|
||||
return buffer.subarray(0, size);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function writeDurable(directory: string, name: string, bytes: Buffer): Promise<void> {
|
||||
const temporary = path.join(directory, `.${name}.${randomUUID()}.tmp`);
|
||||
const handle = await fs.open(temporary, 'wx', 0o600);
|
||||
try {
|
||||
try {
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await fs.rename(temporary, path.join(directory, name));
|
||||
await syncEmbeddingJobDirectory(directory);
|
||||
} finally {
|
||||
await fs.rm(temporary, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function readSlotMetadata(root: string): Promise<RepoMeta | null> {
|
||||
try {
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
bytes = await boundedRead(path.join(root, INDEX_METADATA_FILE), MAX_METADATA_BYTES);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
bytes = await boundedRead(path.join(root, LEGACY_METADATA_FILE), MAX_METADATA_BYTES);
|
||||
}
|
||||
const metadata: RepoMeta = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
||||
if (!metadata || typeof metadata.repoPath !== 'string')
|
||||
throw new Error('Invalid graph slot metadata; artifacts retained');
|
||||
return metadata;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const metadataDigest = (metadata: RepoMeta | null): string | null =>
|
||||
metadata === null ? null : hash(JSON.stringify(metadata));
|
||||
|
||||
/** Replaces the existing pre-write dirty stamp, never an extra post-write stamp.
|
||||
* The coordinator holds the slot lock and must await this before mutating native
|
||||
* data. An interrupted in-place attempt requires the usual dirty-graph rebuild.
|
||||
*/
|
||||
export async function beginInPlaceAnalysis(
|
||||
directory: string,
|
||||
metadata: RepoMeta,
|
||||
saveMetadata: AnalysisPublicationActions['saveMetadata'],
|
||||
): Promise<RepoMeta> {
|
||||
const root = await embeddingJobDirectory(directory);
|
||||
if (
|
||||
!metadata.incrementalInProgress ||
|
||||
(metadata.graphPath !== undefined && !PRIVATE_GRAPH.test(metadata.graphPath))
|
||||
)
|
||||
throw new Error('In-place publication requires a private graph and a dirty write plan');
|
||||
if (await readPreparedAnalysis(root))
|
||||
throw new Error('Recover the pending graph publication before starting another');
|
||||
// Refuse an aliased native target before the caller can perform its first write.
|
||||
await regularFile(path.join(root, metadata.graphPath ?? 'lbug'));
|
||||
const prior = await readSlotMetadata(root);
|
||||
if (prior && prior.repoPath !== metadata.repoPath)
|
||||
throw new Error('Graph slot belongs to another repository');
|
||||
const attemptId = randomUUID().replaceAll('-', '');
|
||||
const dirty: RepoMeta = {
|
||||
...metadata,
|
||||
vectorGeneration: preserveVectorCoverage(metadata, attemptId),
|
||||
incrementalInProgress: {
|
||||
...metadata.incrementalInProgress,
|
||||
writePlan: 'in-place',
|
||||
attemptId,
|
||||
},
|
||||
};
|
||||
delete dirty.graphGeneration;
|
||||
await saveMetadata(dirty);
|
||||
await syncPublishedMetadata(root, dirty);
|
||||
return dirty;
|
||||
}
|
||||
|
||||
/** Call after graph/FTS verification and a successful native checkpoint, before
|
||||
* native close. A close crash leaves the owned build and its publication intent.
|
||||
* No model or vector work may start until publishPreparedAnalysis completes.
|
||||
*/
|
||||
export async function prepareAnalysisPublication(
|
||||
directory: string,
|
||||
buildPath: string,
|
||||
metadata: RepoMeta,
|
||||
coveredContent: string,
|
||||
options: AnalysisPublicationOptions = {},
|
||||
): Promise<PreparedAnalysisPublication> {
|
||||
const root = await embeddingJobDirectory(directory);
|
||||
if (
|
||||
!DIGEST.test(coveredContent) ||
|
||||
!metadata.repoPath ||
|
||||
!metadata.runnerIdentity ||
|
||||
metadata.incrementalInProgress ||
|
||||
metadata.graphWriteCollapsed
|
||||
)
|
||||
throw new Error('Graph publication requires verified source and analyzer metadata');
|
||||
const mode = options.mode ?? 'staged';
|
||||
const sourceFile = path.basename(buildPath);
|
||||
if (
|
||||
(await fs.realpath(path.dirname(buildPath))) !== root ||
|
||||
(mode === 'staged'
|
||||
? !STAGING.test(sourceFile)
|
||||
: sourceFile !== 'lbug' && !PRIVATE_GRAPH.test(sourceFile))
|
||||
)
|
||||
throw new Error('Graph build is outside its owned staging directory');
|
||||
if (!(await regularFile(buildPath))) throw new Error('Graph build is missing');
|
||||
if (await regularFile(path.join(root, RECEIPT)))
|
||||
throw new Error('Recover the pending graph publication before preparing another');
|
||||
const prior = await readSlotMetadata(root);
|
||||
if (prior && prior.repoPath !== metadata.repoPath)
|
||||
throw new Error('Graph slot belongs to another repository');
|
||||
if (
|
||||
options.mode === 'in-place' &&
|
||||
(!GENERATION.test(options.attemptId) ||
|
||||
prior?.incrementalInProgress?.attemptId !== options.attemptId ||
|
||||
(prior.graphPath ?? 'lbug') !== sourceFile ||
|
||||
prior.graphGeneration)
|
||||
)
|
||||
throw new Error('In-place graph attempt is missing or superseded; artifacts retained');
|
||||
const id = options.mode === 'in-place' ? options.attemptId : randomUUID().replaceAll('-', '');
|
||||
const graphFile =
|
||||
mode === 'in-place' && sourceFile !== 'lbug' ? sourceFile : `lbug.generation.${id}`;
|
||||
const metadataFile = `analysis-meta-${id}.json`;
|
||||
const preparedMeta: RepoMeta = {
|
||||
...metadata,
|
||||
vectorGeneration: preserveVectorCoverage(metadata, id),
|
||||
graphPath: graphFile,
|
||||
graphGeneration: { version: 1, id, coveredContent },
|
||||
};
|
||||
const bytes = Buffer.from(JSON.stringify(preparedMeta));
|
||||
if (bytes.length > MAX_METADATA_BYTES)
|
||||
throw new Error('Graph metadata exceeds the supported publication size');
|
||||
const receipt: PreparedAnalysisPublication = {
|
||||
version: 2,
|
||||
mode,
|
||||
id,
|
||||
stagingFile: path.basename(buildPath),
|
||||
graphFile,
|
||||
metadataFile,
|
||||
metadataBytes: bytes.length,
|
||||
metadataDigest: hash(bytes),
|
||||
priorMetadataDigest: metadataDigest(prior),
|
||||
...(options.waitForProcessExit
|
||||
? {
|
||||
writerPid: process.pid,
|
||||
writerInstance: WRITER_INSTANCE,
|
||||
writerStartTime: readProcStartTime(process.pid) ?? undefined,
|
||||
}
|
||||
: {}),
|
||||
...(mode === 'staged' && (await regularFile(path.join(root, 'lbug')))
|
||||
? { legacyGraphFile: `lbug.generation.${randomUUID().replaceAll('-', '')}` }
|
||||
: {}),
|
||||
};
|
||||
if (receipt.legacyGraphFile && prior && !prior.graphPath)
|
||||
receipt.redirectedMetadataDigest = hash(
|
||||
JSON.stringify({ ...prior, graphPath: receipt.legacyGraphFile }),
|
||||
);
|
||||
validateReceipt(receipt);
|
||||
await writeDurable(root, metadataFile, bytes);
|
||||
const payload = JSON.stringify(receipt);
|
||||
await writeDurable(
|
||||
root,
|
||||
RECEIPT,
|
||||
Buffer.from(JSON.stringify({ checksum: hash(payload), payload: receipt })),
|
||||
);
|
||||
return receipt;
|
||||
}
|
||||
|
||||
export async function readPreparedAnalysis(
|
||||
directory: string,
|
||||
): Promise<PreparedAnalysisPublication | undefined> {
|
||||
const root = await embeddingJobDirectory(directory);
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
bytes = await boundedRead(path.join(root, RECEIPT), MAX_RECEIPT_BYTES);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
throw error;
|
||||
}
|
||||
const envelope: unknown = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
||||
if (
|
||||
!envelope ||
|
||||
typeof envelope !== 'object' ||
|
||||
!('payload' in envelope) ||
|
||||
!('checksum' in envelope) ||
|
||||
hash(JSON.stringify(envelope.payload)) !== envelope.checksum
|
||||
)
|
||||
throw new Error('Prepared publication checksum mismatch; artifacts retained');
|
||||
validateReceipt(envelope.payload);
|
||||
return envelope.payload;
|
||||
}
|
||||
|
||||
export async function syncPublishedMetadata(root: string, metadata: RepoMeta): Promise<void> {
|
||||
const metadataPath = path.join(root, INDEX_METADATA_FILE);
|
||||
const saved = await boundedRead(metadataPath, MAX_METADATA_BYTES);
|
||||
if (
|
||||
hash(JSON.stringify(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(saved)))) !==
|
||||
hash(JSON.stringify(metadata))
|
||||
)
|
||||
throw new Error(
|
||||
'Published graph metadata does not match the prepared generation; artifacts retained',
|
||||
);
|
||||
// Windows FlushFileBuffers requires a writable descriptor. The coordinator
|
||||
// owns the write lock; this opens the file without changing its contents.
|
||||
const handle = await fs.open(metadataPath, constants.O_RDWR | (constants.O_NOFOLLOW ?? 0));
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
}
|
||||
|
||||
/** Older binaries ignore private generation pointers. Remove the obsolete
|
||||
* canonical name before advancing metadata, retaining its inode for readers
|
||||
* and rollback. A hard-link failure leaves the old index readable and aborts;
|
||||
* relocating vectors must never trigger a full graph copy.
|
||||
*/
|
||||
async function retireLegacyGraph(
|
||||
root: string,
|
||||
receipt: PreparedAnalysisPublication,
|
||||
metadata: RepoMeta,
|
||||
actions: AnalysisPublicationActions,
|
||||
previous: RepoMeta | null,
|
||||
): Promise<void> {
|
||||
const legacy = path.join(root, 'lbug');
|
||||
if (!receipt.legacyGraphFile) {
|
||||
if (await regularFile(legacy))
|
||||
throw new Error('Unexpected legacy graph appeared during publication');
|
||||
return;
|
||||
}
|
||||
const retired = path.join(root, receipt.legacyGraphFile);
|
||||
if (!(await regularFile(legacy))) {
|
||||
if (!(await regularFile(retired))) throw new Error('Legacy graph retirement artifact missing');
|
||||
return;
|
||||
}
|
||||
if (await regularFile(retired)) {
|
||||
const [before, after] = await Promise.all([fs.stat(legacy), fs.stat(retired)]);
|
||||
if (!before.ino || before.ino !== after.ino || before.dev !== after.dev)
|
||||
throw new Error('Conflicting legacy graph retirement artifact');
|
||||
} else {
|
||||
await fs.link(legacy, retired);
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
}
|
||||
if (previous && previous.repoPath !== metadata.repoPath)
|
||||
throw new Error('Legacy graph belongs to another repository');
|
||||
if (previous && !previous.graphPath) {
|
||||
const redirected = { ...previous, graphPath: receipt.legacyGraphFile };
|
||||
await actions.saveMetadata(redirected);
|
||||
await syncPublishedMetadata(root, redirected);
|
||||
}
|
||||
try {
|
||||
await fs.unlink(legacy);
|
||||
} catch (error) {
|
||||
if (['EBUSY', 'EPERM', 'EACCES'].includes((error as NodeJS.ErrnoException).code ?? ''))
|
||||
throw new Error(
|
||||
'Cannot retire the legacy graph while a reader holds it. Stop older MCP/serve processes and rerun analyze; the prepared graph and embeddings are retained.',
|
||||
{ cause: error },
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
}
|
||||
|
||||
/** Also recovers crashes after rename or metadata publication. Every attempt
|
||||
* re-verifies the owned native artifact; a generation identifier is not proof.
|
||||
* Staged writes retain the prior reader artifact. In-place writes have already
|
||||
* invalidated it and publish a new logical generation on the same private file.
|
||||
*/
|
||||
export async function publishPreparedAnalysis(
|
||||
directory: string,
|
||||
actions: AnalysisPublicationActions,
|
||||
): Promise<RepoMeta | undefined> {
|
||||
const root = await embeddingJobDirectory(directory);
|
||||
const receipt = await readPreparedAnalysis(root);
|
||||
if (!receipt) return undefined;
|
||||
if (receipt.writerPid !== undefined) {
|
||||
const deadline = performance.now() + 2000;
|
||||
for (;;) {
|
||||
// A live, recycled PID is not the process that prepared this receipt.
|
||||
if (
|
||||
receipt.writerPid === process.pid &&
|
||||
receipt.writerInstance &&
|
||||
receipt.writerInstance !== WRITER_INSTANCE
|
||||
)
|
||||
break;
|
||||
const started = readProcStartTime(receipt.writerPid);
|
||||
if (receipt.writerStartTime && started && receipt.writerStartTime !== started) break;
|
||||
try {
|
||||
process.kill(receipt.writerPid, 0);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ESRCH') break;
|
||||
throw new Error(
|
||||
`Cannot verify graph writer PID ${receipt.writerPid}: ${error instanceof Error ? error.message : String(error)}. Run analyze as the original index owner after that process exits; artifacts retained.`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
if (performance.now() >= deadline)
|
||||
throw new Error(
|
||||
`Graph writer PID ${receipt.writerPid} has not exited; publication must wait for physical release. Stop that analysis process and retry; artifacts retained.`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
const bytes = await boundedRead(path.join(root, receipt.metadataFile), receipt.metadataBytes);
|
||||
if (bytes.length !== receipt.metadataBytes || hash(bytes) !== receipt.metadataDigest)
|
||||
throw new Error('Prepared graph metadata checksum mismatch; artifacts retained');
|
||||
const metadata = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as RepoMeta;
|
||||
if (
|
||||
metadata.graphPath !== receipt.graphFile ||
|
||||
metadata.graphGeneration?.version !== 1 ||
|
||||
metadata.graphGeneration.id !== receipt.id ||
|
||||
!DIGEST.test(metadata.graphGeneration.coveredContent)
|
||||
)
|
||||
throw new Error('Prepared graph generation mismatch');
|
||||
const previous = await readSlotMetadata(root);
|
||||
const currentDigest = metadataDigest(previous);
|
||||
if (
|
||||
currentDigest !== receipt.priorMetadataDigest &&
|
||||
currentDigest !== receipt.redirectedMetadataDigest &&
|
||||
currentDigest !== receipt.metadataDigest
|
||||
)
|
||||
throw new Error(
|
||||
'Prepared graph publication was superseded; artifacts retained; retry analysis',
|
||||
);
|
||||
const staged = path.join(root, receipt.stagingFile);
|
||||
const published = path.join(root, receipt.graphFile);
|
||||
const stagedExists = await regularFile(staged);
|
||||
const publishedExists = await regularFile(published);
|
||||
const retainedInPlace = staged === published;
|
||||
if (retainedInPlace ? !publishedExists : stagedExists === publishedExists)
|
||||
throw new Error('Prepared publication has missing or conflicting graph artifacts');
|
||||
const source = stagedExists ? staged : published;
|
||||
await actions.verify(source, metadata);
|
||||
if (stagedExists && !retainedInPlace) {
|
||||
await fs.rename(staged, published);
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
}
|
||||
await retireLegacyGraph(root, receipt, metadata, actions, previous);
|
||||
await actions.saveMetadata(metadata);
|
||||
// The canonical pointer must be durable before callers start paid work.
|
||||
await syncPublishedMetadata(root, metadata);
|
||||
await fs.unlink(path.join(root, RECEIPT));
|
||||
// Persist removal of the recovery pointer before removing what it names.
|
||||
// A power loss must not resurrect a receipt whose metadata is already gone.
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
await fs.unlink(path.join(root, receipt.metadataFile));
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
await reclaimAnalysisArtifacts(root, metadata, previous?.graphPath ?? receipt.legacyGraphFile);
|
||||
return metadata;
|
||||
}
|
||||
293
gitnexus/src/core/analysis-resource-budget.ts
Normal file
293
gitnexus/src/core/analysis-resource-budget.ts
Normal file
|
|
@ -0,0 +1,293 @@
|
|||
import { heapCapMbFor } from './ingestion/utils/effective-ram.js';
|
||||
/** Pure admission policy. Reservations are not an OS-enforced RSS limit. */
|
||||
const MIB = 1024 * 1024;
|
||||
const GRAPH_POOL_MB = 256;
|
||||
const VECTOR_POOL_MB = 64;
|
||||
const MIN_MAIN_HEAP_MB = 384;
|
||||
const MIN_WORKER_HEAP_MB = 512;
|
||||
const WORKER_NATIVE_MB = 128;
|
||||
|
||||
export interface AnalysisResourceRequest {
|
||||
/** Effective process/container RAM, before reserving host headroom. */
|
||||
effectiveMemoryBytes: number;
|
||||
availableCpus: number;
|
||||
/** Explicit total reservation ceiling; independent of the heap-only flag. */
|
||||
totalMb?: number;
|
||||
mainHeapMb?: number;
|
||||
/** Honor an existing heap override when no explicit family ceiling was set.
|
||||
* This may reserve virtual memory above RAM; native defaults still use RAM. */
|
||||
allowHeapOvercommit?: boolean;
|
||||
workers?: number;
|
||||
workerHeapMb?: number;
|
||||
embeddingThreads?: number;
|
||||
/** Existing automatic worker policy; explicit workers still wins. */
|
||||
preferredWorkers?: number;
|
||||
/** Resume/sync can admit embeddings without reserving parser workers. */
|
||||
includeParsing?: boolean;
|
||||
graphPoolBytes?: number;
|
||||
/** LadybugDB COPY accounting uses OS-page-sized granules. */
|
||||
graphPoolGranuleRatio?: number;
|
||||
/** Resident parent process, charged once per child phase. */
|
||||
supervisorBytes?: number;
|
||||
/** Estimated peak model/sidecar allocation; zero for an HTTP provider. */
|
||||
modelMemoryMb: number;
|
||||
}
|
||||
|
||||
export interface AnalysisResourceBudget {
|
||||
readonly totalBytes: number;
|
||||
readonly automatic: boolean;
|
||||
readonly headroomBytes: number;
|
||||
readonly supervisorBytes: number;
|
||||
readonly mainHeapMb: number;
|
||||
readonly parse: {
|
||||
readonly workers: number;
|
||||
readonly workerHeapMb: number;
|
||||
readonly workerNativeBytes: number;
|
||||
readonly graphPoolBytes: number;
|
||||
/** Graph writes run after parser workers drain and may use the full CPU allowance. */
|
||||
readonly nativeThreads: number;
|
||||
readonly reservedBytes: number;
|
||||
};
|
||||
readonly embedding: {
|
||||
readonly graphPoolBytes: number;
|
||||
readonly vectorPoolBytes: number;
|
||||
readonly modelBytes: number;
|
||||
/** Sidecar old-generation allocation, included in modelBytes with native/young space. */
|
||||
readonly modelHeapMb: number;
|
||||
/** Combined source, expanded text, encoded request, vectors and prefetch. */
|
||||
readonly queueBytes: number;
|
||||
readonly nativeThreads: number;
|
||||
readonly reservedBytes: number;
|
||||
};
|
||||
}
|
||||
|
||||
function integer(value: number, label: string, minimum = 1): number {
|
||||
if (!Number.isSafeInteger(value) || value < minimum) {
|
||||
throw new Error(`${label} must be a safe integer of at least ${minimum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function bytes(mb: number, label: string): number {
|
||||
return integer(integer(mb, label, 0) * MIB, `${label} in bytes`, 0);
|
||||
}
|
||||
|
||||
export const UNBOUNDED_GRAPH_POOL_MESSAGE =
|
||||
"Zero graph pool bytes selects LadybugDB's native 80%-of-RAM default, which bounded analysis cannot reserve. " +
|
||||
'Unset GITNEXUS_LBUG_BUFFER_POOL_SIZE for automatic sizing or set a positive byte limit of at least 8388608.';
|
||||
|
||||
/** Resolve before opening a writable database or starting parser/model work. */
|
||||
export function resolveAnalysisResourceBudget(
|
||||
request: AnalysisResourceRequest,
|
||||
): AnalysisResourceBudget {
|
||||
if (request.graphPoolBytes === 0) throw new Error(UNBOUNDED_GRAPH_POOL_MESSAGE);
|
||||
const memory = integer(request.effectiveMemoryBytes, 'effectiveMemoryBytes');
|
||||
const cpus = integer(request.availableCpus, 'availableCpus');
|
||||
const embeddingThreads =
|
||||
request.embeddingThreads === undefined
|
||||
? Math.min(4, cpus)
|
||||
: integer(request.embeddingThreads, 'embedding threads');
|
||||
if (embeddingThreads > cpus) throw new Error('Embedding threads exceed the CPU allowance');
|
||||
const availableMb = Math.floor(memory / MIB);
|
||||
const automatic = request.totalMb === undefined;
|
||||
let totalMb =
|
||||
request.totalMb === undefined
|
||||
? Math.floor(availableMb * 0.75)
|
||||
: integer(request.totalMb, 'resource budget');
|
||||
if (totalMb > availableMb) {
|
||||
throw new Error(
|
||||
`Resource budget ${totalMb} MiB exceeds effective process memory ${availableMb} MiB`,
|
||||
);
|
||||
}
|
||||
const headroomMb = Math.max(128, Math.ceil(totalMb * 0.12));
|
||||
const supervisorMb = Math.ceil(
|
||||
integer(request.supervisorBytes ?? 0, 'supervisor reservation', 0) / MIB,
|
||||
);
|
||||
const poolScale = integer(request.graphPoolGranuleRatio ?? 1, 'graph pool granule ratio');
|
||||
const graphPoolBytes =
|
||||
request.graphPoolBytes === undefined
|
||||
? bytes(
|
||||
automatic
|
||||
? Math.min(2048 * poolScale, Math.floor(availableMb * 0.8))
|
||||
: Math.max(GRAPH_POOL_MB * poolScale, Math.min(4096, Math.floor(totalMb * 0.15))),
|
||||
'graph pool',
|
||||
)
|
||||
: integer(request.graphPoolBytes, 'graph pool bytes', 8 * MIB);
|
||||
// Zero is the native engine's unbounded override, never a bounded allocation.
|
||||
const graphPoolMb = Math.ceil(graphPoolBytes / MIB);
|
||||
const vectorPoolMb = Math.max(VECTOR_POOL_MB, Math.min(4096, Math.floor(totalMb / 8)));
|
||||
const modelMb = integer(request.modelMemoryMb, 'model reservation', 0);
|
||||
if (modelMb > 0 && modelMb < 512)
|
||||
throw new Error(
|
||||
'Local model reservation must be at least 512 MiB for heap and native allocations',
|
||||
);
|
||||
// A supported 10 MiB source node needs its full text/AST working set. Larger
|
||||
// explicit budgets must be able to admit it without truncating its content.
|
||||
const queueMb = Math.max(16, Math.min(512, Math.floor(totalMb / 32)));
|
||||
const embeddingFixedMb =
|
||||
headroomMb + supervisorMb + graphPoolMb + vectorPoolMb + modelMb + queueMb;
|
||||
const includeParsing = request.includeParsing !== false;
|
||||
const cpuWorkers = Math.max(1, cpus - 1);
|
||||
const desiredWorkers =
|
||||
request.workers === undefined
|
||||
? Math.min(integer(request.preferredWorkers ?? 4, 'preferred workers'), cpuWorkers)
|
||||
: integer(request.workers, 'parser workers');
|
||||
const explicitWorkerHeapMb =
|
||||
request.workerHeapMb === undefined
|
||||
? undefined
|
||||
: integer(request.workerHeapMb, 'worker heap reservation');
|
||||
const automaticWorkerHeapMb =
|
||||
explicitWorkerHeapMb ??
|
||||
Math.min(4096, Math.max(MIN_WORKER_HEAP_MB, Math.floor(availableMb / 2 / desiredWorkers)));
|
||||
const parseFixedMb =
|
||||
headroomMb +
|
||||
supervisorMb +
|
||||
graphPoolMb +
|
||||
(includeParsing ? MIN_WORKER_HEAP_MB + WORKER_NATIVE_MB : 0);
|
||||
let maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb);
|
||||
const mainHeapMb =
|
||||
request.mainHeapMb === undefined
|
||||
? automatic
|
||||
? heapCapMbFor(request.effectiveMemoryBytes)
|
||||
: Math.min(16_384, maximumMainMb, Math.max(MIN_MAIN_HEAP_MB, Math.floor(totalMb * 0.35)))
|
||||
: integer(request.mainHeapMb, 'main heap reservation');
|
||||
// Automatic sizing preserves the established independent V8 ceilings: the existing RAM-aware
|
||||
// main heap, half RAM shared by parser heaps. Their finite caps
|
||||
// can sum above RAM; they are not simultaneous resident allocations. Only an
|
||||
// explicit --resource-budget imposes a sum ceiling. A heap override must not
|
||||
// silently consume worker capacity or inflate the native pools/queues.
|
||||
if (automatic) {
|
||||
if (
|
||||
mainHeapMb > availableMb &&
|
||||
!request.allowHeapOvercommit &&
|
||||
request.mainHeapMb !== undefined
|
||||
)
|
||||
throw new Error(
|
||||
'Resource budget cannot admit a main heap above effective memory without an explicit heap override',
|
||||
);
|
||||
totalMb = Math.max(
|
||||
totalMb,
|
||||
mainHeapMb + embeddingFixedMb,
|
||||
headroomMb +
|
||||
supervisorMb +
|
||||
graphPoolMb +
|
||||
mainHeapMb +
|
||||
(includeParsing ? desiredWorkers * (automaticWorkerHeapMb + WORKER_NATIVE_MB) : 0),
|
||||
);
|
||||
maximumMainMb = totalMb - Math.max(embeddingFixedMb, parseFixedMb);
|
||||
}
|
||||
const totalBytes = bytes(totalMb, 'resource budget');
|
||||
// --memory-budget has an established 200 MiB minimum. Preserve that explicit
|
||||
// choice; the larger floor is only the automatic sizing policy.
|
||||
const minimumMainMb = request.mainHeapMb === undefined ? MIN_MAIN_HEAP_MB : 200;
|
||||
if (mainHeapMb < minimumMainMb || mainHeapMb > maximumMainMb) {
|
||||
const minimum =
|
||||
Math.max(minimumMainMb, request.mainHeapMb ?? 0) + Math.max(embeddingFixedMb, parseFixedMb);
|
||||
throw new Error(
|
||||
`Resource budget ${totalMb} MiB cannot admit parser/model work; requires at least ${minimum} MiB for these allocations`,
|
||||
);
|
||||
}
|
||||
|
||||
// Explicit worker counts intentionally permit CPU oversubscription, as the
|
||||
// parser's existing override does. Reserve every worker's heap/native memory
|
||||
// below; automatic concurrency remains bounded by the usable CPU count.
|
||||
const workerBudgetMb = totalMb - headroomMb - supervisorMb - graphPoolMb - mainHeapMb;
|
||||
if (
|
||||
includeParsing &&
|
||||
explicitWorkerHeapMb !== undefined &&
|
||||
explicitWorkerHeapMb < MIN_WORKER_HEAP_MB
|
||||
) {
|
||||
throw new Error(`Worker heap reservation must be at least ${MIN_WORKER_HEAP_MB} MiB`);
|
||||
}
|
||||
const workers = !includeParsing
|
||||
? 0
|
||||
: automatic
|
||||
? desiredWorkers
|
||||
: request.workers === undefined
|
||||
? Math.min(
|
||||
desiredWorkers,
|
||||
Math.floor(
|
||||
workerBudgetMb / ((explicitWorkerHeapMb ?? MIN_WORKER_HEAP_MB) + WORKER_NATIVE_MB),
|
||||
),
|
||||
)
|
||||
: desiredWorkers;
|
||||
const workerHeapMb = !includeParsing
|
||||
? 0
|
||||
: automatic
|
||||
? automaticWorkerHeapMb
|
||||
: (explicitWorkerHeapMb ??
|
||||
Math.min(4096, Math.floor(workerBudgetMb / Math.max(1, workers)) - WORKER_NATIVE_MB));
|
||||
const parseReservedMb =
|
||||
headroomMb +
|
||||
supervisorMb +
|
||||
graphPoolMb +
|
||||
mainHeapMb +
|
||||
workers * (workerHeapMb + WORKER_NATIVE_MB);
|
||||
if (
|
||||
(includeParsing && (workers < 1 || workerHeapMb < MIN_WORKER_HEAP_MB)) ||
|
||||
parseReservedMb > totalMb
|
||||
) {
|
||||
throw new Error(
|
||||
`Parser workers do not fit resource budget ${totalMb} MiB; reduce workers or explicit heap reservations`,
|
||||
);
|
||||
}
|
||||
const embeddingReservedMb = mainHeapMb + embeddingFixedMb;
|
||||
return Object.freeze({
|
||||
totalBytes,
|
||||
automatic,
|
||||
headroomBytes: bytes(headroomMb, 'headroom'),
|
||||
supervisorBytes: bytes(supervisorMb, 'supervisor reservation'),
|
||||
mainHeapMb,
|
||||
parse: Object.freeze({
|
||||
workers,
|
||||
workerHeapMb,
|
||||
workerNativeBytes: bytes(WORKER_NATIVE_MB, 'worker native reservation'),
|
||||
graphPoolBytes,
|
||||
nativeThreads: cpus,
|
||||
reservedBytes: bytes(parseReservedMb, 'parse reservation'),
|
||||
}),
|
||||
embedding: Object.freeze({
|
||||
graphPoolBytes,
|
||||
vectorPoolBytes: bytes(vectorPoolMb, 'vector pool'),
|
||||
modelBytes: bytes(modelMb, 'model reservation'),
|
||||
modelHeapMb: modelMb === 0 ? 0 : Math.min(1024, Math.floor(modelMb / 4)),
|
||||
queueBytes: bytes(queueMb, 'embedding queues'),
|
||||
nativeThreads: embeddingThreads,
|
||||
reservedBytes: bytes(embeddingReservedMb, 'embedding reservation'),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** Account retained payloads before admission, including a prefetched unit. */
|
||||
export class EmbeddingByteBudget {
|
||||
private retained = 0;
|
||||
private peak = 0;
|
||||
|
||||
constructor(readonly limit: number) {
|
||||
integer(limit, 'embedding byte budget');
|
||||
}
|
||||
|
||||
get usedBytes(): number {
|
||||
return this.retained;
|
||||
}
|
||||
get peakBytes(): number {
|
||||
return this.peak;
|
||||
}
|
||||
|
||||
reserve(size: number): () => void {
|
||||
integer(size, 'retained payload bytes', 0);
|
||||
if (size > this.limit - this.retained) {
|
||||
throw new Error(
|
||||
`Embedding payload requires ${size} bytes with ${this.limit - this.retained} bytes available`,
|
||||
);
|
||||
}
|
||||
this.retained += size;
|
||||
this.peak = Math.max(this.peak, this.retained);
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) throw new Error('Embedding byte reservation released twice');
|
||||
released = true;
|
||||
this.retained -= size;
|
||||
};
|
||||
}
|
||||
}
|
||||
7
gitnexus/src/core/analysis-supervision.ts
Normal file
7
gitnexus/src/core/analysis-supervision.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
/** Raised before native graph work when a CLI needs its phase supervisor. */
|
||||
export class AnalysisSupervisorRequired extends Error {
|
||||
constructor() {
|
||||
super('Graph work requires an analysis supervisor');
|
||||
this.name = 'AnalysisSupervisorRequired';
|
||||
}
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ import {
|
|||
requireRegisteredStoragePath,
|
||||
STATUS_STORAGE_REQUIREMENTS,
|
||||
} from '../../storage/storage-resolver.js';
|
||||
import { LBUG_DIRECTORY } from '../../storage/storage-constants.js';
|
||||
import { BRANCHES_DIR, branchSlug } from '../../storage/branch-index.js';
|
||||
import { getCurrentBranch } from '../../storage/git.js';
|
||||
import { escapeCypherString } from '../lbug/cypher-escape.js';
|
||||
|
|
@ -85,10 +84,7 @@ async function findRepoForCwd(cwd: string): Promise<{
|
|||
return {
|
||||
name: bestMatch.name,
|
||||
storagePath,
|
||||
lbugPath:
|
||||
indexDir === storagePath
|
||||
? resolveGraphPath(storagePath)
|
||||
: path.join(indexDir, LBUG_DIRECTORY),
|
||||
lbugPath: resolveGraphPath(indexDir),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -4,8 +4,18 @@ import { createRequire } from 'node:module';
|
|||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js';
|
||||
import type { WorkerMessage } from '../../server/analyze-worker-protocol.js';
|
||||
import type {
|
||||
WorkerMessage,
|
||||
ContinueMessage,
|
||||
GraphPreparedMessage,
|
||||
} from '../../server/analyze-worker-protocol.js';
|
||||
import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js';
|
||||
import {
|
||||
analysisMayEmbed,
|
||||
analysisSupervisorReservation,
|
||||
resolveHostAnalysisResources,
|
||||
} from '../analysis-host-resources.js';
|
||||
import { projectPreparedAnalysisHandoff } from '../analysis-phase-supervisor.js';
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
export type AutoSyncAnalysisRunner = (
|
||||
|
|
@ -20,6 +30,7 @@ export type AutoSyncAnalysisRunner = (
|
|||
) => Promise<Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>>;
|
||||
|
||||
interface AnalysisWorker extends Pick<ChildProcess, 'send' | 'on'> {
|
||||
pid?: number;
|
||||
stdout?: Pick<NodeJS.ReadableStream, 'resume'> | null;
|
||||
stderr?: Pick<NodeJS.ReadableStream, 'resume'> | null;
|
||||
unref?: () => void;
|
||||
|
|
@ -95,8 +106,26 @@ export function createAutoSyncAnalysisRunner(
|
|||
overrides: Partial<AutoSyncAnalysisLaunchDeps> = {},
|
||||
): AutoSyncAnalysisRunner {
|
||||
const deps = { ...DEFAULT_DEPS, ...overrides };
|
||||
return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency, onProgress) =>
|
||||
new Promise<Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>>((resolve, reject) => {
|
||||
return async (
|
||||
repoPath,
|
||||
options,
|
||||
timeoutMs,
|
||||
signal,
|
||||
onCancellationRequested,
|
||||
concurrency,
|
||||
onProgress,
|
||||
) => {
|
||||
signal?.throwIfAborted();
|
||||
const resources =
|
||||
options.resources ??
|
||||
resolveHostAnalysisResources({
|
||||
mainHeapMb: resolveWorkerHeapMb(concurrency) + 48,
|
||||
includeEmbeddings: await analysisMayEmbed(repoPath, options),
|
||||
supervisorBytes: Math.ceil(analysisSupervisorReservation() / (concurrency ?? 1)),
|
||||
concurrency,
|
||||
});
|
||||
const analysisOptions = { ...options, graphPublication: 'auto' as const, resources };
|
||||
return new Promise<Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>>((resolve, reject) => {
|
||||
if (signal?.aborted) {
|
||||
reject(new Error('Analysis cancelled.'));
|
||||
return;
|
||||
|
|
@ -112,17 +141,16 @@ export function createAutoSyncAnalysisRunner(
|
|||
reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`));
|
||||
return;
|
||||
}
|
||||
const workerHeapMb = resolveWorkerHeapMb(concurrency);
|
||||
const workerHeapMb = resources.mainHeapMb - 48;
|
||||
const execArgv = isDev
|
||||
? [
|
||||
'--import',
|
||||
pathToFileURL(_require.resolve('tsx/esm')).href,
|
||||
`--max-old-space-size=${workerHeapMb}`,
|
||||
'--max-semi-space-size=16',
|
||||
]
|
||||
: [`--max-old-space-size=${workerHeapMb}`];
|
||||
const child = deps.forkWorker(workerPath, execArgv);
|
||||
child.stdout?.resume();
|
||||
child.stderr?.resume();
|
||||
: [`--max-old-space-size=${workerHeapMb}`, '--max-semi-space-size=16'];
|
||||
let child: AnalysisWorker;
|
||||
|
||||
let terminalOutcome: WorkerMessage | undefined;
|
||||
let terminationError: Error | undefined;
|
||||
|
|
@ -138,8 +166,8 @@ export function createAutoSyncAnalysisRunner(
|
|||
// so both handles have to go. Never a kill: the child may be inside
|
||||
// native work and is left to reach its own safe point.
|
||||
const releaseChild = () => {
|
||||
child.channel?.unref?.();
|
||||
child.unref?.();
|
||||
child?.channel?.unref?.();
|
||||
child?.unref?.();
|
||||
};
|
||||
const settle = (
|
||||
error?: Error,
|
||||
|
|
@ -159,7 +187,7 @@ export function createAutoSyncAnalysisRunner(
|
|||
// IPC has the same semantics on macOS and Windows. The worker exits only
|
||||
// after reaching a JS-visible safe point; this parent keeps ownership until then.
|
||||
try {
|
||||
child.send({ type: 'cancel' });
|
||||
child?.send({ type: 'cancel' });
|
||||
} catch {
|
||||
// A closed IPC channel still has an exit/error path. Do not force-kill a
|
||||
// worker that may be inside native code.
|
||||
|
|
@ -192,72 +220,122 @@ export function createAutoSyncAnalysisRunner(
|
|||
const onAbort = () => requestCancellation(new Error('Analysis cancelled.'));
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
|
||||
child.on('message', (message: WorkerMessage) => {
|
||||
// Once timeout/cancellation requested shutdown, its reason owns the
|
||||
// result. A terminal IPC can already be queued behind cancellation.
|
||||
if (terminalOutcome || terminationError) return;
|
||||
if (message.type === 'progress') {
|
||||
// The worker also sends its log lines as progress with percent -1;
|
||||
// those are not progress. A listener's failure must not decide the
|
||||
// run's outcome.
|
||||
if (message.percent < 0) return;
|
||||
try {
|
||||
onProgress?.(message.percent, message.message);
|
||||
} catch {
|
||||
// ignored
|
||||
const launch = (continuation?: ContinueMessage) => {
|
||||
child = deps.forkWorker(workerPath, execArgv);
|
||||
const phaseChild = child;
|
||||
let prepared: GraphPreparedMessage | undefined;
|
||||
child.stdout?.resume();
|
||||
child.stderr?.resume();
|
||||
child.on('message', (message: WorkerMessage) => {
|
||||
// Once timeout/cancellation requested shutdown, its reason owns the
|
||||
// result. A terminal IPC can already be queued behind cancellation.
|
||||
if (terminalOutcome || terminationError || settled || child !== phaseChild) return;
|
||||
if (message.type === 'progress') {
|
||||
// Worker log messages use percent -1 and do not represent progress.
|
||||
if (message.percent < 0) return;
|
||||
try {
|
||||
onProgress?.(message.percent, message.message);
|
||||
} catch {
|
||||
// A listener failure must not decide the run's outcome.
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.type === 'graph-ready') {
|
||||
// Watch has no serving reader cache. Publication is already verified.
|
||||
try {
|
||||
child.send({ type: 'graph-visible' });
|
||||
} catch (error) {
|
||||
requestCancellation(error as Error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.type === 'graph-prepared') {
|
||||
try {
|
||||
if (prepared || continuation) throw new Error('Repeated graph preparation');
|
||||
prepared = { ...message, handoff: projectPreparedAnalysisHandoff(message.handoff) };
|
||||
} catch (error) {
|
||||
requestCancellation(error as Error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
terminalOutcome = message;
|
||||
deps.clearTimeoutFn(timeout);
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`);
|
||||
requestCancellation(workerError);
|
||||
// This settles immediately rather than waiting out the grace, so the
|
||||
// grace timer that would otherwise have released the child is cleared
|
||||
// by cleanup(). Release it here instead — an errored channel does not
|
||||
// mean the worker stopped.
|
||||
releaseChild();
|
||||
settle(workerError);
|
||||
});
|
||||
child.on('exit', (code, childSignal) => {
|
||||
if (settled || child !== phaseChild) return;
|
||||
if (terminationError) {
|
||||
settle(terminationError);
|
||||
return;
|
||||
}
|
||||
if (terminalOutcome?.type === 'complete') {
|
||||
if (code === 0 && !childSignal) {
|
||||
const { stats, graphWriteCollapsed } = terminalOutcome.result;
|
||||
settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats });
|
||||
} else
|
||||
settle(
|
||||
new Error(
|
||||
`Auto-sync analyze worker exited unsuccessfully (${childSignal ?? code}).`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!terminalOutcome && prepared && code === 0 && !childSignal && phaseChild.pid) {
|
||||
try {
|
||||
launch({
|
||||
type: 'continue',
|
||||
repoPath,
|
||||
options: analysisOptions,
|
||||
writerPid: phaseChild.pid,
|
||||
prepared,
|
||||
});
|
||||
} catch (error) {
|
||||
settle(error as Error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (terminalOutcome?.type === 'error') {
|
||||
settle(
|
||||
new AutoSyncAnalysisError(terminalOutcome.message, {
|
||||
code: terminalOutcome.code,
|
||||
retryable: terminalOutcome.retryable,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
terminalOutcome = message;
|
||||
deps.clearTimeoutFn(timeout);
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`);
|
||||
requestCancellation(workerError);
|
||||
// This settles immediately rather than waiting out the grace, so the
|
||||
// grace timer that would otherwise have released the child is cleared
|
||||
// by cleanup(). Release it here instead — an errored channel does not
|
||||
// mean the worker stopped.
|
||||
releaseChild();
|
||||
settle(workerError);
|
||||
});
|
||||
child.on('exit', (code, childSignal) => {
|
||||
if (settled) return;
|
||||
if (terminationError) {
|
||||
settle(terminationError);
|
||||
return;
|
||||
}
|
||||
if (terminalOutcome?.type === 'complete') {
|
||||
const { stats, graphWriteCollapsed } = terminalOutcome.result;
|
||||
// Only present on a collapsed write, so a healthy result keeps its shape.
|
||||
settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats });
|
||||
return;
|
||||
}
|
||||
if (terminalOutcome?.type === 'error') {
|
||||
settle(
|
||||
new AutoSyncAnalysisError(terminalOutcome.message, {
|
||||
code: terminalOutcome.code,
|
||||
retryable: terminalOutcome.retryable,
|
||||
}),
|
||||
new Error(
|
||||
`Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
});
|
||||
try {
|
||||
child.send(continuation ?? { type: 'start', repoPath, options: analysisOptions });
|
||||
} catch (error) {
|
||||
const startError = new Error(
|
||||
`Failed to start auto-sync analyze worker: ${(error as Error).message}`,
|
||||
);
|
||||
requestCancellation(startError);
|
||||
releaseChild();
|
||||
settle(startError);
|
||||
}
|
||||
settle(
|
||||
new Error(
|
||||
`Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`,
|
||||
),
|
||||
);
|
||||
});
|
||||
};
|
||||
try {
|
||||
child.send({ type: 'start', repoPath, options });
|
||||
launch();
|
||||
} catch (error) {
|
||||
const startError = new Error(
|
||||
`Failed to start auto-sync analyze worker: ${(error as Error).message}`,
|
||||
);
|
||||
requestCancellation(startError);
|
||||
settle(startError);
|
||||
settle(error as Error);
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner();
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import {
|
|||
resolveLanguageKey,
|
||||
} from '../tree-sitter/parser-loader.js';
|
||||
import { parseSourceSafe } from '../tree-sitter/safe-parse.js';
|
||||
import { getLanguageForFileContent, getProvider } from '../ingestion/languages/index.js';
|
||||
import { extractNotebookPython, isNotebookPath } from '../ingestion/ipynb-extractor.js';
|
||||
|
||||
const parserCache = new Map<string, any>();
|
||||
|
|
@ -20,6 +19,8 @@ const parserCache = new Map<string, any>();
|
|||
* Returns null if language is unavailable or parsing fails.
|
||||
*/
|
||||
export const ensureAndParse = async (content: string, filePath: string): Promise<any | null> => {
|
||||
const { getLanguageForFileContent, getProvider } =
|
||||
await import('../ingestion/languages/index.js');
|
||||
// Same classifier as ingest. Filename-only maps `.h` → C++, so Objective-C
|
||||
// headers (and method snippets from those headers) would parse with the
|
||||
// wrong grammar and miss class_interface / protocol_declaration / methods.
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
* Character-based sliding window chunking (pure, no tree-sitter dependency)
|
||||
*/
|
||||
|
||||
import { buildLineIndex, resolveChunkLines } from './line-index.js';
|
||||
|
||||
export interface Chunk {
|
||||
text: string;
|
||||
chunkIndex: number;
|
||||
|
|
@ -13,14 +11,71 @@ export interface Chunk {
|
|||
endLine: number;
|
||||
}
|
||||
|
||||
/** Sliding windows with constant auxiliary memory, including line tracking. */
|
||||
export function* iterateCharacterChunks(
|
||||
content: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize = 1200,
|
||||
overlap = 120,
|
||||
): Generator<Chunk> {
|
||||
if (
|
||||
!Number.isSafeInteger(chunkSize) ||
|
||||
chunkSize < 1 ||
|
||||
!Number.isSafeInteger(overlap) ||
|
||||
overlap < 0
|
||||
)
|
||||
throw new Error('Invalid character chunk size or overlap');
|
||||
if (content.length <= chunkSize) {
|
||||
yield {
|
||||
text: content,
|
||||
chunkIndex: 0,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
startLine,
|
||||
endLine,
|
||||
};
|
||||
return;
|
||||
}
|
||||
let offset = 0;
|
||||
let chunkIndex = 0;
|
||||
let relativeStart = 0;
|
||||
let relativeEnd = 0;
|
||||
let nextStartNewline = content.indexOf('\n');
|
||||
let nextEndNewline = nextStartNewline;
|
||||
while (offset < content.length) {
|
||||
const end = Math.min(offset + chunkSize, content.length);
|
||||
while (nextStartNewline >= 0 && nextStartNewline < offset) {
|
||||
relativeStart++;
|
||||
nextStartNewline = content.indexOf('\n', nextStartNewline + 1);
|
||||
}
|
||||
while (nextEndNewline >= 0 && nextEndNewline < end - 1) {
|
||||
relativeEnd++;
|
||||
nextEndNewline = content.indexOf('\n', nextEndNewline + 1);
|
||||
}
|
||||
yield {
|
||||
text: content.slice(offset, end),
|
||||
chunkIndex: chunkIndex++,
|
||||
startOffset: offset,
|
||||
endOffset: end,
|
||||
startLine: startLine + relativeStart,
|
||||
endLine: startLine + relativeEnd,
|
||||
};
|
||||
offset = end - overlap;
|
||||
if (offset >= content.length || end >= content.length) break;
|
||||
if (offset <= (chunkIndex > 1 ? end - chunkSize : 0)) offset = end;
|
||||
}
|
||||
}
|
||||
|
||||
/** Compatibility API for callers that need all chunks. */
|
||||
export const characterChunk = (
|
||||
content: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number = 1200,
|
||||
overlap: number = 120,
|
||||
chunkSize = 1200,
|
||||
overlap = 120,
|
||||
): Chunk[] => {
|
||||
if (content.length <= chunkSize) {
|
||||
if (content.length <= chunkSize)
|
||||
return [
|
||||
{
|
||||
text: content,
|
||||
|
|
@ -31,33 +86,5 @@ export const characterChunk = (
|
|||
endLine,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const chunks: Chunk[] = [];
|
||||
let offset = 0;
|
||||
const lineOffsets = buildLineIndex(content);
|
||||
|
||||
while (offset < content.length) {
|
||||
const end = Math.min(offset + chunkSize, content.length);
|
||||
const chunkText = content.slice(offset, end);
|
||||
const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine);
|
||||
|
||||
chunks.push({
|
||||
text: chunkText,
|
||||
chunkIndex: chunks.length,
|
||||
startOffset: offset,
|
||||
endOffset: end,
|
||||
startLine: lineRange.startLine,
|
||||
endLine: lineRange.endLine,
|
||||
});
|
||||
|
||||
offset = end - overlap;
|
||||
if (offset >= content.length) break;
|
||||
if (end >= content.length) break;
|
||||
if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) {
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
return [...iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap)];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
export { type Chunk, characterChunk } from './character-chunk.js';
|
||||
|
||||
import { characterChunk } from './character-chunk.js';
|
||||
import { iterateCharacterChunks } from './character-chunk.js';
|
||||
import type { Chunk } from './character-chunk.js';
|
||||
import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js';
|
||||
import { buildLineIndex, resolveChunkLines } from './line-index.js';
|
||||
|
|
@ -81,10 +81,68 @@ export const chunkNode = async (
|
|||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number = 1200,
|
||||
overlap: number = 120,
|
||||
chunkSize = 1200,
|
||||
overlap = 120,
|
||||
options?: ChunkOptions,
|
||||
): Promise<EmbeddingChunk[]> => {
|
||||
if (label !== 'Class' && content.length <= chunkSize)
|
||||
return [
|
||||
{
|
||||
text: content,
|
||||
chunkIndex: 0,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
startLine,
|
||||
endLine,
|
||||
},
|
||||
];
|
||||
return [
|
||||
...(await prepareNodeChunks(
|
||||
label,
|
||||
content,
|
||||
filePath,
|
||||
startLine,
|
||||
endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
options,
|
||||
)),
|
||||
];
|
||||
};
|
||||
|
||||
/** Produce one chunk at a time, without retaining generated text for the node. */
|
||||
export async function* iterateNodeChunks(
|
||||
label: string,
|
||||
content: string,
|
||||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize = 1200,
|
||||
overlap = 120,
|
||||
options?: ChunkOptions,
|
||||
): AsyncGenerator<EmbeddingChunk> {
|
||||
yield* await prepareNodeChunks(
|
||||
label,
|
||||
content,
|
||||
filePath,
|
||||
startLine,
|
||||
endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
async function prepareNodeChunks(
|
||||
label: string,
|
||||
content: string,
|
||||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number,
|
||||
overlap: number,
|
||||
options?: ChunkOptions,
|
||||
): Promise<Iterable<EmbeddingChunk>> {
|
||||
// Even a short class contains implementations. Never fall back to raw Class
|
||||
// source: an unavailable parser yields identity/name metadata only.
|
||||
if (label === 'Class') {
|
||||
|
|
@ -105,28 +163,17 @@ export const chunkNode = async (
|
|||
return [classMetadataChunk(startLine, endLine)];
|
||||
}
|
||||
}
|
||||
// Content fits in one chunk — no splitting needed
|
||||
if (content.length <= chunkSize) {
|
||||
return [
|
||||
{
|
||||
text: content,
|
||||
chunkIndex: 0,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
startLine,
|
||||
endLine,
|
||||
},
|
||||
];
|
||||
return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap);
|
||||
}
|
||||
|
||||
const rule = CHUNKING_RULES[label];
|
||||
if (!rule || options?.strategy === 'fixed') {
|
||||
return characterChunk(content, startLine, endLine, chunkSize, overlap);
|
||||
return iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap);
|
||||
}
|
||||
|
||||
let chunks: Generator<Chunk> | undefined;
|
||||
try {
|
||||
if (rule.mode === CHUNK_MODE_AST_FUNCTION) {
|
||||
const astChunks = await astChunk(
|
||||
if (rule?.mode === CHUNK_MODE_AST_FUNCTION) {
|
||||
chunks = await astChunk(
|
||||
content,
|
||||
filePath,
|
||||
startLine,
|
||||
|
|
@ -136,11 +183,8 @@ export const chunkNode = async (
|
|||
rule,
|
||||
options?.name,
|
||||
);
|
||||
if (astChunks.length > 0) return astChunks;
|
||||
}
|
||||
|
||||
if (rule.mode === CHUNK_MODE_AST_DECLARATION) {
|
||||
const declarationChunks = await declarationChunk(
|
||||
} else if (rule?.mode === CHUNK_MODE_AST_DECLARATION) {
|
||||
chunks = await declarationChunk(
|
||||
content,
|
||||
filePath,
|
||||
startLine,
|
||||
|
|
@ -149,15 +193,13 @@ export const chunkNode = async (
|
|||
overlap,
|
||||
rule,
|
||||
);
|
||||
if (declarationChunks.length > 0) return declarationChunks;
|
||||
}
|
||||
} catch {
|
||||
// AST parsing failed — fall through to character fallback
|
||||
// Only preparation may fall back. Never restart after yielding a chunk:
|
||||
// the consumer may already have durably stored it.
|
||||
}
|
||||
|
||||
// Character-based fallback for everything else
|
||||
return characterChunk(content, startLine, endLine, chunkSize, overlap);
|
||||
};
|
||||
return chunks ?? iterateCharacterChunks(content, startLine, endLine, chunkSize, overlap);
|
||||
}
|
||||
|
||||
/**
|
||||
* AST-based chunking for Function/Method
|
||||
|
|
@ -173,9 +215,9 @@ const astChunk = async (
|
|||
overlap: number,
|
||||
rule: ChunkingRule,
|
||||
expectedName?: string,
|
||||
): Promise<Chunk[]> => {
|
||||
): Promise<Generator<Chunk> | undefined> => {
|
||||
const tree = await ensureAndParse(content, filePath);
|
||||
if (!tree) return [];
|
||||
if (!tree) return undefined;
|
||||
|
||||
const root = tree.rootNode;
|
||||
const lineOffsets = buildLineIndex(content);
|
||||
|
|
@ -183,11 +225,11 @@ const astChunk = async (
|
|||
// Find the function/method declaration in the snippet AST.
|
||||
// tree-sitter parses node.content (a snippet), so rows are relative (0-based).
|
||||
const targetNode = findFunctionNode(root, expectedName);
|
||||
if (!targetNode) return [];
|
||||
if (!targetNode) return undefined;
|
||||
|
||||
// Get the body (statements) via childForFieldName('body')
|
||||
const bodyNode = targetNode.childForFieldName('body');
|
||||
if (!bodyNode) return [];
|
||||
if (!bodyNode) return undefined;
|
||||
|
||||
// Extract individual statements
|
||||
const statements: Array<{ startIndex: number; endIndex: number }> = [];
|
||||
|
|
@ -208,7 +250,7 @@ const astChunk = async (
|
|||
}
|
||||
}
|
||||
|
||||
if (statements.length === 0) return [];
|
||||
if (statements.length === 0) return undefined;
|
||||
|
||||
// A fitting statement must not fall into character splitting just because
|
||||
// its function header or closing delimiters consume the remaining budget.
|
||||
|
|
@ -397,18 +439,18 @@ const declarationChunk = async (
|
|||
chunkSize: number,
|
||||
overlap: number,
|
||||
rule: ChunkingRule,
|
||||
): Promise<Chunk[]> => {
|
||||
): Promise<Generator<Chunk> | undefined> => {
|
||||
const tree = await ensureAndParse(content, filePath);
|
||||
if (!tree) return [];
|
||||
if (!tree) return undefined;
|
||||
|
||||
const targetNode = findDeclarationNode(tree.rootNode);
|
||||
if (!targetNode) return [];
|
||||
if (!targetNode) return undefined;
|
||||
|
||||
const bodyNode = getDeclarationBodyNode(targetNode);
|
||||
if (!bodyNode) return [];
|
||||
if (!bodyNode) return undefined;
|
||||
|
||||
const members = collectDeclarationUnits(bodyNode, rule.groupFields);
|
||||
if (members.length === 0) return [];
|
||||
if (members.length === 0) return undefined;
|
||||
|
||||
return chunkByUnits(
|
||||
content,
|
||||
|
|
@ -443,7 +485,7 @@ const buildChunk = (
|
|||
};
|
||||
};
|
||||
|
||||
const chunkByUnits = (
|
||||
function* chunkByUnits(
|
||||
content: string,
|
||||
lineOffsets: Int32Array,
|
||||
baseStartLine: number,
|
||||
|
|
@ -454,8 +496,8 @@ const chunkByUnits = (
|
|||
containerEndOffset: number,
|
||||
includeContainerPrefixOnFirstChunk: boolean,
|
||||
includeContainerSuffixOnLastChunk: boolean,
|
||||
): Chunk[] => {
|
||||
const chunks: Chunk[] = [];
|
||||
): Generator<Chunk> {
|
||||
let chunkIndex = 0;
|
||||
let chunkStartUnitIdx = 0;
|
||||
|
||||
while (chunkStartUnitIdx < units.length) {
|
||||
|
|
@ -496,32 +538,31 @@ const chunkByUnits = (
|
|||
oversizedEndOffset,
|
||||
baseStartLine,
|
||||
);
|
||||
const oversizedChunks = characterChunk(
|
||||
for (const chunk of iterateCharacterChunks(
|
||||
content.slice(oversizedStartOffset, oversizedEndOffset),
|
||||
oversizedLineRange.startLine,
|
||||
oversizedLineRange.endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
).map((chunk, offsetIdx) => ({
|
||||
...chunk,
|
||||
chunkIndex: chunks.length + offsetIdx,
|
||||
startOffset: chunk.startOffset + oversizedStartOffset,
|
||||
endOffset: chunk.endOffset + oversizedStartOffset,
|
||||
}));
|
||||
chunks.push(...oversizedChunks);
|
||||
)) {
|
||||
yield {
|
||||
...chunk,
|
||||
chunkIndex: chunkIndex++,
|
||||
startOffset: chunk.startOffset + oversizedStartOffset,
|
||||
endOffset: chunk.endOffset + oversizedStartOffset,
|
||||
};
|
||||
}
|
||||
chunkStartUnitIdx += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
chunks.push(
|
||||
buildChunk(
|
||||
content,
|
||||
lineOffsets,
|
||||
chunks.length,
|
||||
chunkStartOffset,
|
||||
candidateEndOffset,
|
||||
baseStartLine,
|
||||
),
|
||||
yield buildChunk(
|
||||
content,
|
||||
lineOffsets,
|
||||
chunkIndex++,
|
||||
chunkStartOffset,
|
||||
candidateEndOffset,
|
||||
baseStartLine,
|
||||
);
|
||||
|
||||
if (chunkEndUnitIdx === units.length - 1) {
|
||||
|
|
@ -540,9 +581,7 @@ const chunkByUnits = (
|
|||
chunkStartUnitIdx = nextChunkStartUnitIdx;
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
};
|
||||
}
|
||||
|
||||
const findOverlapStartIndex = (
|
||||
statements: Array<{ startIndex: number; endIndex: number }>,
|
||||
|
|
@ -654,7 +693,7 @@ const projectClassChunks = (
|
|||
overlap: number,
|
||||
strategy: EmbeddingConfig['chunkStrategy'] = 'ast',
|
||||
expectedName?: string,
|
||||
): EmbeddingChunk[] => {
|
||||
): Iterable<EmbeddingChunk> => {
|
||||
const target = tree && findDeclarationNode(tree.rootNode, expectedName);
|
||||
const body = target && getDeclarationBodyNode(target);
|
||||
if (!body || target.hasError) return [classMetadataChunk(startLine, endLine)];
|
||||
|
|
@ -745,7 +784,7 @@ const projectClassChunks = (
|
|||
const projected = pieces.join('');
|
||||
const chunks =
|
||||
strategy === 'fixed'
|
||||
? characterChunk(projected, 1, 1, chunkSize, overlap)
|
||||
? iterateCharacterChunks(projected, 1, 1, chunkSize, overlap)
|
||||
: chunkByUnits(
|
||||
projected,
|
||||
buildLineIndex(projected),
|
||||
|
|
@ -772,17 +811,19 @@ const projectClassChunks = (
|
|||
segment.source + Math.max(0, Math.min(offset - segment.start, segment.end - segment.start))
|
||||
);
|
||||
};
|
||||
return chunks.map((chunk) => {
|
||||
const startOffset = sourceOffset(chunk.startOffset);
|
||||
const endOffset = sourceOffset(chunk.endOffset - 1) + 1;
|
||||
return {
|
||||
...chunk,
|
||||
startOffset,
|
||||
endOffset,
|
||||
...resolveChunkLines(originalLines, startOffset, endOffset, startLine),
|
||||
classDeclaration: header,
|
||||
};
|
||||
});
|
||||
return (function* () {
|
||||
for (const chunk of chunks) {
|
||||
const startOffset = sourceOffset(chunk.startOffset);
|
||||
const endOffset = sourceOffset(chunk.endOffset - 1) + 1;
|
||||
yield {
|
||||
...chunk,
|
||||
startOffset,
|
||||
endOffset,
|
||||
...resolveChunkLines(originalLines, startOffset, endOffset, startLine),
|
||||
classDeclaration: header,
|
||||
};
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
const collectDeclarationUnits = (
|
||||
|
|
|
|||
129
gitnexus/src/core/embeddings/embedding-inputs.ts
Normal file
129
gitnexus/src/core/embeddings/embedding-inputs.ts
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
/** Shared streaming input layout for generation and historical-vector validation. */
|
||||
import { isNotebookPath } from '../ingestion/ipynb-extractor.js';
|
||||
import { ensureAndParse } from './ast-utils.js';
|
||||
import { iterateNodeChunks, type EmbeddingChunk } from './chunker.js';
|
||||
import { extractStructuralNames } from './structural-extractor.js';
|
||||
import {
|
||||
generateEmbeddingText,
|
||||
splitEmbeddingText,
|
||||
withoutOverlappingContext,
|
||||
} from './text-generator.js';
|
||||
import { needsEmbeddingSource, type EmbeddingSourceContext } from './source-reader.js';
|
||||
import {
|
||||
isShortLabel,
|
||||
STRUCTURAL_LABELS,
|
||||
type EmbeddableNode,
|
||||
type EmbeddingConfig,
|
||||
} from './types.js';
|
||||
|
||||
export interface EmbeddingInput {
|
||||
text: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingReuseOptions {
|
||||
node: EmbeddableNode;
|
||||
prepareInputValidation?: () => Promise<void>;
|
||||
inputs?: () => AsyncIterable<EmbeddingInput>;
|
||||
matchesHash?: (stored: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export async function* iterateEmbeddingInputs(
|
||||
indexedNode: EmbeddableNode,
|
||||
config: EmbeddingConfig,
|
||||
options: {
|
||||
source?: EmbeddingSourceContext;
|
||||
readSource?: (node: EmbeddableNode) => Promise<{ content: string; release: () => void }>;
|
||||
signal?: AbortSignal;
|
||||
} = {},
|
||||
): AsyncGenerator<EmbeddingInput> {
|
||||
const hydrate =
|
||||
options.source?.contentRetention === 'full' && needsEmbeddingSource(indexedNode, config);
|
||||
if (hydrate && !options.readSource)
|
||||
throw new Error('Original embedding source needs a budgeted reader');
|
||||
const lease = hydrate ? await options.readSource?.(indexedNode) : undefined;
|
||||
// Never retain expanded source or structural metadata in the fetched unit.
|
||||
const node = { ...indexedNode, content: lease?.content ?? indexedNode.content };
|
||||
try {
|
||||
options.signal?.throwIfAborted();
|
||||
let classTree: Awaited<ReturnType<typeof ensureAndParse>> | undefined;
|
||||
if (node.label === 'Class') {
|
||||
try {
|
||||
classTree = await ensureAndParse(node.content, node.filePath);
|
||||
} catch {
|
||||
classTree = null;
|
||||
}
|
||||
}
|
||||
if (!isShortLabel(node.label) && STRUCTURAL_LABELS.has(node.label)) {
|
||||
try {
|
||||
const names = await extractStructuralNames(
|
||||
node.content,
|
||||
node.filePath,
|
||||
classTree,
|
||||
node.label === 'Class' ? node.name : undefined,
|
||||
);
|
||||
node.methodNames = names.methodNames;
|
||||
node.fieldNames = names.fieldNames;
|
||||
} catch {
|
||||
/* Structural enrichment remains optional. */
|
||||
}
|
||||
}
|
||||
const startLine = node.startLine ?? 0,
|
||||
endLine = node.endLine ?? 0;
|
||||
const chunks: Iterable<EmbeddingChunk> | AsyncIterable<EmbeddingChunk> = isShortLabel(
|
||||
node.label,
|
||||
)
|
||||
? [
|
||||
{
|
||||
text: node.content,
|
||||
chunkIndex: 0,
|
||||
startLine,
|
||||
endLine,
|
||||
startOffset: 0,
|
||||
endOffset: node.content.length,
|
||||
},
|
||||
]
|
||||
: iterateNodeChunks(
|
||||
node.label,
|
||||
node.content,
|
||||
node.filePath,
|
||||
startLine,
|
||||
endLine,
|
||||
config.chunkSize,
|
||||
config.overlap,
|
||||
{ strategy: config.chunkStrategy, name: node.name, tree: classTree },
|
||||
);
|
||||
const conservativeRange =
|
||||
options.source &&
|
||||
(options.source.contentRetention !== 'full' || isNotebookPath(node.filePath));
|
||||
let prevTail = '',
|
||||
previousEndOffset: number | undefined;
|
||||
for await (const chunk of chunks) {
|
||||
options.signal?.throwIfAborted();
|
||||
node.classDeclaration = chunk.classDeclaration;
|
||||
const text = generateEmbeddingText(
|
||||
node,
|
||||
chunk.text,
|
||||
config,
|
||||
chunk.chunkIndex,
|
||||
withoutOverlappingContext(prevTail, previousEndOffset, chunk.startOffset),
|
||||
);
|
||||
for (const part of splitEmbeddingText(text, config.maxInputChars)) {
|
||||
// Detach slices so queued inputs cannot pin a large declaration/source.
|
||||
yield {
|
||||
text:
|
||||
lease || text.length > config.maxInputChars
|
||||
? Buffer.from(part, 'utf16le').toString('utf16le')
|
||||
: part,
|
||||
startLine: conservativeRange ? startLine : chunk.startLine,
|
||||
endLine: conservativeRange ? endLine : chunk.endLine,
|
||||
};
|
||||
}
|
||||
prevTail = config.overlap > 0 ? chunk.text.slice(-config.overlap) : '';
|
||||
previousEndOffset = chunk.endOffset;
|
||||
}
|
||||
} finally {
|
||||
lease?.release();
|
||||
}
|
||||
}
|
||||
80
gitnexus/src/core/embeddings/embedding-job-identity.ts
Normal file
80
gitnexus/src/core/embeddings/embedding-job-identity.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type { EmbeddingIdentity } from './embedding-identity.js';
|
||||
import type { EmbeddingConfig } from './types.js';
|
||||
import type { ContentRetention } from '../../storage/repo-meta.js';
|
||||
|
||||
export interface EmbeddingGraphIdentity {
|
||||
generation: string;
|
||||
coveredContent: string;
|
||||
revision: string;
|
||||
analyzer: string;
|
||||
parser: string;
|
||||
schema: string;
|
||||
vectorSchema: string;
|
||||
}
|
||||
|
||||
const digest = (value: unknown): string =>
|
||||
createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
|
||||
/** Explicit fields exclude credentials and scheduling settings from persisted identity. */
|
||||
export function embeddingJobIdentity(
|
||||
graph: EmbeddingGraphIdentity,
|
||||
vector: EmbeddingIdentity,
|
||||
textVersion: string,
|
||||
config: Pick<
|
||||
EmbeddingConfig,
|
||||
'chunkSize' | 'overlap' | 'maxSnippetLength' | 'maxDescriptionLength' | 'maxInputChars'
|
||||
> &
|
||||
Partial<Pick<EmbeddingConfig, 'chunkStrategy'>>,
|
||||
contentRetention: ContentRetention = 'full',
|
||||
): { identity: string; sourceIdentity: string; vectorIdentity: string; inputIdentity: string } {
|
||||
const source = [
|
||||
graph.generation,
|
||||
graph.coveredContent,
|
||||
graph.revision,
|
||||
graph.analyzer,
|
||||
graph.parser,
|
||||
graph.schema,
|
||||
graph.vectorSchema,
|
||||
];
|
||||
if (
|
||||
source.some((value) => typeof value !== 'string' || !value) ||
|
||||
!vector.model ||
|
||||
!/^(local|http:[a-f0-9]{64})$/.test(vector.provider) ||
|
||||
!Number.isSafeInteger(vector.dimensions) ||
|
||||
vector.dimensions < 1 ||
|
||||
!textVersion ||
|
||||
![
|
||||
config.chunkSize,
|
||||
config.overlap,
|
||||
config.maxSnippetLength,
|
||||
config.maxDescriptionLength,
|
||||
config.maxInputChars,
|
||||
].every((n) => Number.isSafeInteger(n) && n >= 0) ||
|
||||
config.chunkSize < 1 ||
|
||||
config.overlap >= config.chunkSize ||
|
||||
config.maxInputChars < 2 ||
|
||||
!['ast', 'fixed'].includes(config.chunkStrategy ?? 'ast') ||
|
||||
!['full', 'symbol', 'none'].includes(contentRetention)
|
||||
)
|
||||
throw new Error('Incomplete embedding job identity');
|
||||
const sourceIdentity = digest(source);
|
||||
const vectorIdentity = digest([vector.provider, vector.model, vector.dimensions]);
|
||||
const inputIdentity = digest([
|
||||
'source-aware-ast-inputs:v1',
|
||||
textVersion,
|
||||
config.chunkStrategy ?? 'ast',
|
||||
contentRetention,
|
||||
config.chunkSize,
|
||||
config.overlap,
|
||||
config.maxSnippetLength,
|
||||
config.maxDescriptionLength,
|
||||
config.maxInputChars,
|
||||
]);
|
||||
return {
|
||||
sourceIdentity,
|
||||
vectorIdentity,
|
||||
inputIdentity,
|
||||
identity: digest([1, sourceIdentity, vectorIdentity, inputIdentity]),
|
||||
};
|
||||
}
|
||||
604
gitnexus/src/core/embeddings/embedding-job-runner.ts
Normal file
604
gitnexus/src/core/embeddings/embedding-job-runner.ts
Normal file
|
|
@ -0,0 +1,604 @@
|
|||
/** One durable manifest unit, one bounded provider batch, no corpus-sized maps. */
|
||||
import { createHash } from 'node:crypto';
|
||||
import { EmbeddingByteBudget } from '../analysis-resource-budget.js';
|
||||
import type { VectorNodeReceipt, VectorStore, VectorWriteChunk } from '../lbug/vector-store.js';
|
||||
import {
|
||||
embeddingInputFailureKey,
|
||||
type EmbeddingJobJournal,
|
||||
type EmbeddingJobState,
|
||||
type EmbeddingUnitIntent,
|
||||
} from '../../storage/embedding-job-journal.js';
|
||||
import {
|
||||
fetchEmbeddingWorkUnit,
|
||||
readEmbeddingWorkUnits,
|
||||
type EmbeddingManifestReceipt,
|
||||
type EmbeddingManifestSource,
|
||||
type EmbeddingWorkUnit,
|
||||
} from './embedding-work-manifest.js';
|
||||
import { contentHashForNode, embedInputRanges, isInputRejection } from './embedding-pipeline.js';
|
||||
import { iterateEmbeddingInputs, type EmbeddingReuseOptions } from './embedding-inputs.js';
|
||||
import {
|
||||
EmbeddingSourceReader,
|
||||
needsEmbeddingSource,
|
||||
type EmbeddingSourceContext,
|
||||
} from './source-reader.js';
|
||||
import { resolveEmbeddingConfig } from './config.js';
|
||||
import { type EmbeddingConfig, type EmbeddableNode } from './types.js';
|
||||
import type { embedBatch } from './embedder.js';
|
||||
|
||||
export interface EmbeddingReuseSource {
|
||||
options: { batchBytes: number };
|
||||
requiresInputValidation?: boolean;
|
||||
reusable(
|
||||
nodeId: string,
|
||||
contentHash: string,
|
||||
options: EmbeddingReuseOptions,
|
||||
): Promise<VectorNodeReceipt | null>;
|
||||
readChunks: VectorStore['readChunks'];
|
||||
}
|
||||
|
||||
export interface EmbeddingJobRunnerOptions {
|
||||
directory: string;
|
||||
manifest: EmbeddingManifestReceipt;
|
||||
source: EmbeddingManifestSource;
|
||||
sourceContext?: EmbeddingSourceContext;
|
||||
journal: EmbeddingJobJournal;
|
||||
store: VectorStore;
|
||||
/** The controller has validated vector/input identities before providing reuse. */
|
||||
reuse?: EmbeddingReuseSource;
|
||||
/** The writer was adopted from an older graph. Persist this choice for the
|
||||
* whole job: old complete receipts cannot skip current-input validation. */
|
||||
revalidateStoredInputs?: boolean;
|
||||
config?: Partial<EmbeddingConfig>;
|
||||
queueBytes: number;
|
||||
provider: typeof embedBatch;
|
||||
initializeProvider?: () => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void;
|
||||
}
|
||||
|
||||
export interface EmbeddingJobRunResult {
|
||||
state: EmbeddingJobState;
|
||||
peakQueueBytes: number;
|
||||
/** Absent while failed work remains. Publication is the controller's job. */
|
||||
indexKind?: 'hnsw' | 'exact';
|
||||
}
|
||||
|
||||
/** Caller owns the job lock and closes all handles, including on rejection. */
|
||||
export async function runEmbeddingJob(
|
||||
options: EmbeddingJobRunnerOptions,
|
||||
): Promise<EmbeddingJobRunResult> {
|
||||
const { directory, manifest, source, journal, store, signal } = options;
|
||||
const config = resolveEmbeddingConfig(options.config ?? {});
|
||||
const budget = new EmbeddingByteBudget(options.queueBytes);
|
||||
let sourceReader: EmbeddingSourceReader | undefined;
|
||||
let releaseSourceCache: (() => void) | undefined;
|
||||
const dimensions = store.options.dimensions;
|
||||
// UTF-16 text + worst-case JSON escaping/encoding, bounded HTTP response
|
||||
// decoding/JSON parsing, float results, JS arrays and native copies. HTTP
|
||||
// responses are capped at 4096 + 32*dimensions bytes per row by the controller.
|
||||
// The provider's runtime has a separate reservation.
|
||||
const rowBytes = (textChars: number, idChars: number) =>
|
||||
16 * textChars + 512 * dimensions + 64 * 1024 + 4 * idChars;
|
||||
const minimumBatchBytes = rowBytes(config.maxInputChars, 0);
|
||||
if (minimumBatchBytes > budget.limit)
|
||||
throw new Error('Embedding input/vector batch does not fit the resource budget');
|
||||
let initialized = false;
|
||||
let attempted = 0;
|
||||
let failures = 0;
|
||||
let consecutiveFailures = 0;
|
||||
const remainingNodes =
|
||||
manifest.totalNodes - journal.snapshot().completedNodes - journal.snapshot().reusedNodes;
|
||||
const ratioFloor = Math.min(20, Math.max(5, Math.ceil(remainingNodes / config.subBatchSize / 2)));
|
||||
// Bounded even if every source node is rejected. IDs are needed only until
|
||||
// the existing five-node input-rejection guard fires or a success resets it.
|
||||
const rejected = new Set<string>();
|
||||
const progress = () => options.onProgress?.(journal.snapshot(), budget.peakBytes);
|
||||
|
||||
const processUnit = async (
|
||||
unit: EmbeddingWorkUnit,
|
||||
intent: EmbeddingUnitIntent,
|
||||
): Promise<void> => {
|
||||
signal?.throwIfAborted();
|
||||
const active = journal.snapshot().active ?? intent;
|
||||
const history = active.inputFailures ?? {};
|
||||
const excluded = new Set<string>();
|
||||
if (Object.keys(history).length) {
|
||||
const nodesByKey = new Map(
|
||||
unit.nodes.map((node) => [
|
||||
active.inputFailureKeyFormat === 'sha256' ? embeddingInputFailureKey(node.id) : node.id,
|
||||
node.id,
|
||||
]),
|
||||
);
|
||||
for (const [key, attempts] of Object.entries(history)) {
|
||||
const id = nodesByKey.get(key);
|
||||
if (id === undefined)
|
||||
throw new Error('Embedding rejection history does not match its manifest unit');
|
||||
if (attempts >= 3) excluded.add(id);
|
||||
}
|
||||
}
|
||||
const complete = options.revalidateStoredInputs
|
||||
? new Map<string, VectorNodeReceipt>()
|
||||
: await store.completedNodes(unit.nodes.map((node) => node.id));
|
||||
for (const id of excluded) complete.delete(id);
|
||||
const missing = unit.nodes.filter((node) => !complete.has(node.id) && !excluded.has(node.id));
|
||||
const retainedBytes = missing.reduce(
|
||||
(n, key) => n + 2 * (key.contentBytes + key.metadataBytes),
|
||||
0,
|
||||
);
|
||||
// One node may expand its text/AST while the rest of the fetched unit stays
|
||||
// retained. Admission precedes intent, provider calls and vector mutation.
|
||||
const scratchBytes = missing.reduce(
|
||||
(n, key) => Math.max(n, 16 * (key.contentBytes + key.metadataBytes)),
|
||||
0,
|
||||
);
|
||||
const maxIdChars = missing.reduce((n, key) => Math.max(n, key.id.length), 0);
|
||||
const maximumBatchRows = Math.min(
|
||||
256,
|
||||
config.subBatchSize,
|
||||
Math.floor(store.options.batchBytes / (16 * dimensions + 128 + 2 * (maxIdChars + 40))),
|
||||
);
|
||||
if (maximumBatchRows < 1) throw new Error('Vector write does not fit its batch reservation');
|
||||
const minimumUnitBatchBytes = rowBytes(config.maxInputChars, maxIdChars);
|
||||
const unitBytes = retainedBytes + scratchBytes + minimumUnitBatchBytes;
|
||||
if (releaseSourceCache && budget.usedBytes + unitBytes > budget.limit) {
|
||||
sourceReader = undefined;
|
||||
releaseSourceCache();
|
||||
releaseSourceCache = undefined;
|
||||
}
|
||||
if (budget.usedBytes + unitBytes > budget.limit)
|
||||
throw new Error(
|
||||
`Embedding unit ${unit.ordinal} requires a larger resource budget; no source text was truncated`,
|
||||
);
|
||||
if (!journal.snapshot().active) await journal.begin(intent);
|
||||
if (excluded.size) await store.excludeNodes([...excluded]);
|
||||
progress();
|
||||
if (!missing.length) {
|
||||
if (complete.size) await store.checkpointUnit([...complete.values()]);
|
||||
await journal.acknowledge({
|
||||
reusedNodes: complete.size,
|
||||
chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0),
|
||||
});
|
||||
progress();
|
||||
return;
|
||||
}
|
||||
const fetched = await fetchEmbeddingWorkUnit(
|
||||
source,
|
||||
{ ...unit, nodes: missing, retainedBytes },
|
||||
budget,
|
||||
);
|
||||
const releaseScratch = budget.reserve(scratchBytes);
|
||||
let batch: Array<Omit<VectorWriteChunk, 'embedding'> & { text: string; release: () => void }> =
|
||||
[];
|
||||
// Retained vector copies share one bounded write buffer, whether their
|
||||
// source is a reusable artifact or a newly completed provider request.
|
||||
let copyBatch: VectorWriteChunk[] = [];
|
||||
let copyBytes = 0;
|
||||
let copyReleases: Array<() => void> = [];
|
||||
const generated = new Map<
|
||||
string,
|
||||
{ contentHash: string; chunks: number; ended: boolean; inputsDigest?: string }
|
||||
>();
|
||||
const failed = new Set<string>();
|
||||
const rejectedInputs = new Set<string>();
|
||||
let failureCause: 'input' | 'provider' = 'input';
|
||||
let abort: unknown;
|
||||
let reusedNodes = complete.size;
|
||||
|
||||
const flushCopies = async (): Promise<void> => {
|
||||
if (!copyBatch.length) return;
|
||||
signal?.throwIfAborted();
|
||||
try {
|
||||
await store.appendBatch(copyBatch);
|
||||
} finally {
|
||||
for (const release of copyReleases) release();
|
||||
copyBatch = [];
|
||||
copyBytes = 0;
|
||||
copyReleases = [];
|
||||
}
|
||||
};
|
||||
|
||||
const queueWrites = async (chunks: VectorWriteChunk[]): Promise<void> => {
|
||||
const bytes = chunks.reduce(
|
||||
(total, chunk) => total + 32 * dimensions + 1024 + 4 * chunk.nodeId.length,
|
||||
0,
|
||||
);
|
||||
if (
|
||||
copyBatch.length &&
|
||||
(copyBatch.length + chunks.length > 256 ||
|
||||
copyBytes + bytes > store.options.batchBytes ||
|
||||
budget.usedBytes + bytes > budget.limit)
|
||||
)
|
||||
await flushCopies();
|
||||
// The active provider batch already reserves these vector copies. A
|
||||
// tight queue can write immediately without a second retained reservation.
|
||||
if (bytes > store.options.batchBytes || budget.usedBytes + bytes > budget.limit) {
|
||||
await store.appendBatch(chunks);
|
||||
return;
|
||||
}
|
||||
const release = budget.reserve(bytes);
|
||||
copyBatch.push(...chunks);
|
||||
copyBytes += bytes;
|
||||
copyReleases.push(release);
|
||||
};
|
||||
|
||||
const flush = async (): Promise<void> => {
|
||||
if (!batch.length) return;
|
||||
signal?.throwIfAborted();
|
||||
if (!initialized) {
|
||||
await options.initializeProvider?.();
|
||||
initialized = true;
|
||||
}
|
||||
attempted++;
|
||||
let providerFailed = false;
|
||||
try {
|
||||
for await (const result of embedInputRanges(
|
||||
batch.map((row) => row.text),
|
||||
batch.length,
|
||||
signal,
|
||||
options.provider,
|
||||
)) {
|
||||
signal?.throwIfAborted();
|
||||
const rows = batch.slice(result.start, result.end);
|
||||
if ('error' in result) {
|
||||
for (const row of rows) failed.add(row.nodeId);
|
||||
if (isInputRejection(result.error)) {
|
||||
for (const row of rows) rejectedInputs.add(row.nodeId);
|
||||
for (const row of rows) if (rejected.size < 5) rejected.add(row.nodeId);
|
||||
} else {
|
||||
failureCause = 'provider';
|
||||
providerFailed = true;
|
||||
consecutiveFailures++;
|
||||
if (consecutiveFailures >= 5) abort = result.error;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
result.embeddings.length !== rows.length ||
|
||||
result.embeddings.some(
|
||||
(v) => v.length !== dimensions || v.some((n) => !Number.isFinite(n)),
|
||||
)
|
||||
)
|
||||
throw new Error('Embedding provider returned invalid vector coverage');
|
||||
consecutiveFailures = 0;
|
||||
rejected.clear();
|
||||
const updates = rows.flatMap((row, i) =>
|
||||
failed.has(row.nodeId)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
nodeId: row.nodeId,
|
||||
contentHash: row.contentHash,
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
embedding: Array.from(result.embeddings[i]),
|
||||
},
|
||||
],
|
||||
);
|
||||
if (updates.length) await queueWrites(updates);
|
||||
}
|
||||
if (providerFailed) failures++;
|
||||
if (!abort && providerFailed && attempted >= ratioFloor && failures / attempted >= 0.25)
|
||||
abort = new Error(
|
||||
'Embedding endpoint failure rate exceeded 25%; unfinished units remain resumable',
|
||||
);
|
||||
if (!abort && rejected.size >= 5)
|
||||
abort = new Error(
|
||||
'Embedding inputs rejected for five nodes without a success; unfinished units remain resumable',
|
||||
);
|
||||
} finally {
|
||||
for (const row of batch) row.release();
|
||||
batch = [];
|
||||
}
|
||||
};
|
||||
|
||||
const prepareInputValidation = options.reuse?.requiresInputValidation
|
||||
? async () => {
|
||||
// Release a provider batch only when a matching legacy group needs
|
||||
// input expansion. Ineligible groups keep normal provider batching.
|
||||
await flush();
|
||||
if (abort) throw abort;
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const nodeInputs = (node: EmbeddableNode) =>
|
||||
iterateEmbeddingInputs(node, config, {
|
||||
source: options.sourceContext,
|
||||
signal,
|
||||
readSource: async (indexedNode) => {
|
||||
const sourceContext = options.sourceContext;
|
||||
if (!sourceContext) throw new Error('Missing indexed embedding source context');
|
||||
let release: (() => void) | undefined;
|
||||
try {
|
||||
if (!sourceReader) {
|
||||
const cacheBytes = Math.min(16 * 1024 * 1024, Math.floor(budget.limit / 8));
|
||||
if (budget.usedBytes + cacheBytes + minimumUnitBatchBytes > budget.limit) {
|
||||
await flush();
|
||||
await flushCopies();
|
||||
}
|
||||
if (abort) throw abort;
|
||||
releaseSourceCache = budget.reserve(cacheBytes);
|
||||
sourceReader = new EmbeddingSourceReader(sourceContext, cacheBytes);
|
||||
}
|
||||
const content = await sourceReader.read(indexedNode, async (fileBytes) => {
|
||||
signal?.throwIfAborted();
|
||||
// Whole-file decoding/line index, notebook reconstruction, AST and
|
||||
// projected declarations live until this node's iterator closes.
|
||||
const bytes = 64 * 1024 + 16 * fileBytes;
|
||||
if (budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit) {
|
||||
await flush();
|
||||
await flushCopies();
|
||||
}
|
||||
if (abort) throw abort;
|
||||
if (
|
||||
!Number.isSafeInteger(bytes) ||
|
||||
budget.usedBytes + bytes + minimumUnitBatchBytes > budget.limit
|
||||
)
|
||||
throw new Error(
|
||||
'Original embedding source requires a larger resource budget; no source text was truncated',
|
||||
);
|
||||
release = budget.reserve(bytes);
|
||||
});
|
||||
if (!release) throw new Error('Original embedding source was not admitted');
|
||||
return { content, release };
|
||||
} catch (error) {
|
||||
release?.();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const sourceAware = (node: EmbeddableNode) =>
|
||||
options.sourceContext?.contentRetention === 'full' && needsEmbeddingSource(node, config);
|
||||
const inputHasher = () =>
|
||||
createHash('sha1').update('embedding-inputs:v1\n').update(JSON.stringify(config.modelId));
|
||||
|
||||
try {
|
||||
// Hash once per fetched node; current-job complete groups never fetch/hash.
|
||||
const inputs = fetched.nodes.map((node) => ({
|
||||
nodeId: node.id,
|
||||
contentHash: contentHashForNode(node, config, options.sourceContext),
|
||||
sourceStartLine: node.startLine ?? 0,
|
||||
sourceEndLine: node.endLine ?? 0,
|
||||
}));
|
||||
const reusableNodes = new Map<string, VectorNodeReceipt>();
|
||||
for (let i = 0; i < inputs.length; i++) {
|
||||
const input = inputs[i],
|
||||
node = fetched.nodes[i];
|
||||
const baseHash = input.contentHash;
|
||||
let digest: string | undefined;
|
||||
const matchesHash = async (previous: string): Promise<boolean> => {
|
||||
if (!sourceAware(node) || !/^[a-f0-9]{40}:[a-f0-9]{40}$/.test(previous)) return false;
|
||||
if (previous.startsWith(`${baseHash}:`)) {
|
||||
input.contentHash = previous;
|
||||
return true;
|
||||
}
|
||||
if (!digest) {
|
||||
const hash = inputHasher();
|
||||
for await (const part of nodeInputs(node))
|
||||
hash.update(JSON.stringify([part.text, part.startLine, part.endLine]));
|
||||
digest = hash.digest('hex');
|
||||
}
|
||||
if (!previous.endsWith(`:${digest}`)) return false;
|
||||
input.contentHash = `${baseHash}:${digest}`;
|
||||
return true;
|
||||
};
|
||||
if (options.revalidateStoredInputs) {
|
||||
const retained = await store.reuseNode(input, async (previous) =>
|
||||
(await matchesHash(previous)) ? input.contentHash : undefined,
|
||||
);
|
||||
if (retained) {
|
||||
complete.set(input.nodeId, retained);
|
||||
reusedNodes++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const reusable = await options.reuse?.reusable(node.id, input.contentHash, {
|
||||
node,
|
||||
prepareInputValidation,
|
||||
inputs: () => nodeInputs(node),
|
||||
matchesHash,
|
||||
});
|
||||
if (reusable) reusableNodes.set(node.id, reusable);
|
||||
else input.contentHash = baseHash;
|
||||
}
|
||||
const missingInputs = options.revalidateStoredInputs
|
||||
? inputs.filter((input) => !complete.has(input.nodeId))
|
||||
: inputs;
|
||||
if (missingInputs.length) await store.beginNodes(missingInputs);
|
||||
const copied: Array<{ nodeId: string; contentHash: string; chunks: number }> = [];
|
||||
for (let i = 0; i < fetched.nodes.length; i++) {
|
||||
const node = fetched.nodes[i];
|
||||
const input = inputs[i];
|
||||
signal?.throwIfAborted();
|
||||
if (options.revalidateStoredInputs && complete.has(node.id)) continue;
|
||||
const reuse = options.reuse;
|
||||
const reusable = reusableNodes.get(node.id);
|
||||
if (reusable && reuse) {
|
||||
// Flush before copy so there is only one vector batch in memory.
|
||||
await flush();
|
||||
if (abort) break;
|
||||
const copyRowBytes = 32 * dimensions + 1024 + 4 * node.id.length;
|
||||
for (let start = 0; start < reusable.chunks; ) {
|
||||
if (
|
||||
copyBatch.length &&
|
||||
(copyBatch.length === 256 ||
|
||||
Math.min(store.options.batchBytes - copyBytes, budget.limit - budget.usedBytes) <
|
||||
copyRowBytes)
|
||||
)
|
||||
await flushCopies();
|
||||
const copyRows = Math.min(
|
||||
256 - copyBatch.length,
|
||||
Math.floor(
|
||||
Math.min(
|
||||
reuse.options.batchBytes,
|
||||
store.options.batchBytes - copyBytes,
|
||||
budget.limit - budget.usedBytes,
|
||||
) / copyRowBytes,
|
||||
),
|
||||
);
|
||||
if (copyRows < 1)
|
||||
throw new Error('Reusable vector batch does not fit the resource budget');
|
||||
const length = Math.min(copyRows, reusable.chunks - start);
|
||||
// Reuse has no HTTP body or JSON decoder; retain only vector copies.
|
||||
const bytes = length * copyRowBytes;
|
||||
const release = budget.reserve(bytes);
|
||||
try {
|
||||
const chunks = await reuse.readChunks(reusable, start, length);
|
||||
const delta = input.sourceStartLine - (reusable.sourceStartLine ?? 0);
|
||||
copyBatch.push(
|
||||
...chunks.map((chunk) => ({
|
||||
...chunk,
|
||||
nodeId: node.id,
|
||||
contentHash: input.contentHash,
|
||||
startLine: chunk.startLine + delta,
|
||||
endLine:
|
||||
chunk.endLine === reusable.sourceEndLine
|
||||
? input.sourceEndLine
|
||||
: chunk.endLine + delta,
|
||||
})),
|
||||
);
|
||||
} catch (error) {
|
||||
release();
|
||||
throw error;
|
||||
}
|
||||
copyReleases.push(release);
|
||||
copyBytes += bytes;
|
||||
start += length;
|
||||
}
|
||||
copied.push({ nodeId: node.id, contentHash: input.contentHash, chunks: reusable.chunks });
|
||||
reusedNodes++;
|
||||
continue;
|
||||
}
|
||||
const state = {
|
||||
contentHash: input.contentHash,
|
||||
chunks: 0,
|
||||
ended: false,
|
||||
inputsDigest: undefined as string | undefined,
|
||||
};
|
||||
generated.set(node.id, state);
|
||||
const hash = sourceAware(node) ? inputHasher() : undefined;
|
||||
for await (const generatedInput of nodeInputs(node)) {
|
||||
if (failed.has(node.id) || abort) break;
|
||||
hash?.update(
|
||||
JSON.stringify([generatedInput.text, generatedInput.startLine, generatedInput.endLine]),
|
||||
);
|
||||
const bytes = rowBytes(generatedInput.text.length, node.id.length);
|
||||
if (
|
||||
batch.length &&
|
||||
(batch.length >= maximumBatchRows || budget.usedBytes + bytes > budget.limit)
|
||||
)
|
||||
await flush();
|
||||
if (failed.has(node.id) || abort) break;
|
||||
if (copyBatch.length && budget.usedBytes + bytes > budget.limit) await flushCopies();
|
||||
const release = budget.reserve(bytes);
|
||||
batch.push({
|
||||
...input,
|
||||
chunkIndex: state.chunks++,
|
||||
...generatedInput,
|
||||
release,
|
||||
});
|
||||
}
|
||||
state.ended = !failed.has(node.id) && !abort;
|
||||
if (state.ended) state.inputsDigest = hash?.digest('hex');
|
||||
if (abort) break;
|
||||
}
|
||||
if (!abort) await flush();
|
||||
if (copyBatch.length) await flushCopies();
|
||||
const attempts = await journal.recordInputRejections([...rejectedInputs]);
|
||||
for (const id of rejectedInputs) {
|
||||
if (attempts[id] < 3) continue;
|
||||
await store.excludeNodes([id]);
|
||||
excluded.add(id);
|
||||
generated.delete(id);
|
||||
failed.delete(id);
|
||||
}
|
||||
const finished = [
|
||||
...[...generated]
|
||||
.filter(([id, node]) => node.ended && !failed.has(id))
|
||||
.map(([nodeId, node]) => ({
|
||||
nodeId,
|
||||
contentHash: node.contentHash,
|
||||
chunks: node.chunks,
|
||||
inputsDigest: node.inputsDigest,
|
||||
})),
|
||||
...copied,
|
||||
];
|
||||
if (finished.length)
|
||||
for (const receipt of await store.finishNodes(finished))
|
||||
complete.set(receipt.nodeId, receipt);
|
||||
if (complete.size) {
|
||||
if (options.revalidateStoredInputs)
|
||||
await store.markSelected(journal.snapshot().identity, [...complete.keys()]);
|
||||
await store.checkpointUnit([...complete.values()]);
|
||||
}
|
||||
if (complete.size + excluded.size === unit.nodes.length) {
|
||||
await journal.acknowledge({
|
||||
reusedNodes,
|
||||
chunks: [...complete.values()].reduce((n, r) => n + r.chunks, 0),
|
||||
});
|
||||
} else await journal.fail(failureCause);
|
||||
progress();
|
||||
if (abort) throw abort;
|
||||
} finally {
|
||||
for (const row of batch) row.release();
|
||||
for (const release of copyReleases) release();
|
||||
releaseScratch();
|
||||
fetched.release();
|
||||
}
|
||||
};
|
||||
|
||||
const replay = async (intent: EmbeddingUnitIntent): Promise<void> => {
|
||||
const units = readEmbeddingWorkUnits(directory, manifest, intent.start);
|
||||
try {
|
||||
const first = await units.next();
|
||||
if (
|
||||
first.done ||
|
||||
first.value.next.offset !== intent.next.offset ||
|
||||
first.value.unit.nodes.length !== intent.nodes
|
||||
)
|
||||
throw new Error('Embedding recovery intent does not match the manifest');
|
||||
await processUnit(first.value.unit, intent);
|
||||
} finally {
|
||||
await units.return(undefined);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
if (journal.snapshot().phase === 'embedding') {
|
||||
const active = journal.snapshot().active;
|
||||
if (active) await replay(active);
|
||||
for await (const intent of journal.failed()) await replay(intent);
|
||||
let cursor = journal.snapshot().cursor;
|
||||
for await (const { unit, next } of readEmbeddingWorkUnits(directory, manifest, cursor)) {
|
||||
await processUnit(unit, {
|
||||
start: cursor,
|
||||
next,
|
||||
nodes: unit.nodes.length,
|
||||
origin: 'manifest',
|
||||
});
|
||||
cursor = next;
|
||||
}
|
||||
if (journal.snapshot().failedUnits)
|
||||
return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes };
|
||||
await journal.indexing();
|
||||
progress();
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
if (options.revalidateStoredInputs && !store.isSealed)
|
||||
await store.pruneUnselected(
|
||||
journal.snapshot().identity,
|
||||
manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0),
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
const indexKind = await store.seal(
|
||||
manifest.totalNodes - (journal.snapshot().excludedNodes ?? 0),
|
||||
journal.snapshot().chunks,
|
||||
);
|
||||
return { state: journal.snapshot(), peakQueueBytes: budget.peakBytes, indexKind };
|
||||
} finally {
|
||||
sourceReader = undefined;
|
||||
releaseSourceCache?.();
|
||||
}
|
||||
}
|
||||
626
gitnexus/src/core/embeddings/embedding-job.ts
Normal file
626
gitnexus/src/core/embeddings/embedding-job.ts
Normal file
|
|
@ -0,0 +1,626 @@
|
|||
/** Common lifecycle for jobs on a published graph. The caller holds its index lock. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import type { RepoMeta } from '../../storage/repo-meta.js';
|
||||
import {
|
||||
createEmbeddingJobDirectories,
|
||||
embeddingJobDirectory,
|
||||
syncEmbeddingJobDirectory,
|
||||
readEmbeddingJobRecord,
|
||||
writeEmbeddingJobRecord,
|
||||
} from '../../storage/embedding-job.js';
|
||||
import {
|
||||
adoptEmbeddingArtifact,
|
||||
readEmbeddingArtifactSpace,
|
||||
type EmbeddingArtifactOwner,
|
||||
} from '../../storage/embedding-artifact.js';
|
||||
import {
|
||||
EmbeddingJobJournal,
|
||||
type EmbeddingJobState,
|
||||
} from '../../storage/embedding-job-journal.js';
|
||||
import {
|
||||
assertOwnedVectorGeneration,
|
||||
metadataMatchesGraph,
|
||||
} from '../../storage/vector-generation.js';
|
||||
import type { AnalysisResourceBudget } from '../analysis-resource-budget.js';
|
||||
import { contentRetentionFromMeta } from '../content-retention.js';
|
||||
import { syncPublishedMetadata } from '../analysis-publication.js';
|
||||
import { EmbeddingGraphSource } from '../lbug/embedding-source.js';
|
||||
import { LegacyVectorSource, legacyVectorCompatibility } from '../lbug/legacy-vector-source.js';
|
||||
import { readLegacyEmbeddingSource } from '../../storage/legacy-embedding-source.js';
|
||||
import { drainAnalysisNativeResources } from '../lbug/lbug-adapter.js';
|
||||
import { VectorStore, VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js';
|
||||
import { estimateBufferPool } from '../lbug/lbug-config.js';
|
||||
import { EMBEDDING_TEXT_VERSION } from './embedding-pipeline.js';
|
||||
import { embeddingJobIdentity } from './embedding-job-identity.js';
|
||||
import { resolveEmbeddingIdentity, type EmbeddingIdentity } from './embedding-identity.js';
|
||||
import { resolveEmbeddingConfig } from './config.js';
|
||||
import {
|
||||
createEmbeddingWorkManifest,
|
||||
loadEmbeddingWorkManifest,
|
||||
type EmbeddingManifestSource,
|
||||
} from './embedding-work-manifest.js';
|
||||
import { runEmbeddingJob } from './embedding-job-runner.js';
|
||||
import { publishVectorGeneration, type VectorPublicationActions } from './vector-publication.js';
|
||||
import type { EmbeddingConfig, ModelProgressCallback } from './types.js';
|
||||
import type { embedBatch } from './embedder.js';
|
||||
import type { EmbeddingSidecarReservation } from './embedding-sidecar-client.js';
|
||||
|
||||
type GraphSource = EmbeddingManifestSource & { close(): Promise<void> };
|
||||
export interface PublishedEmbeddingJobOptions {
|
||||
metadataDirectory: string;
|
||||
/** Already resolved/owned graph, pinned for the entire job by the index lock. */
|
||||
graphPath: string;
|
||||
resources: AnalysisResourceBudget['embedding'];
|
||||
config?: Partial<EmbeddingConfig>;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (state: EmbeddingJobState, peakQueueBytes: number) => void;
|
||||
onModelProgress?: ModelProgressCallback;
|
||||
log?: (message: string) => void;
|
||||
metadata: VectorPublicationActions;
|
||||
}
|
||||
|
||||
/** Provider/source seams allow deterministic native-storage restart tests. */
|
||||
export interface PublishedEmbeddingJobDependencies {
|
||||
identity?: () => EmbeddingIdentity;
|
||||
openSource?: (file: string, poolBytes: number, threads: number) => Promise<GraphSource>;
|
||||
provider?: typeof embedBatch;
|
||||
initializeProvider?: (config: EmbeddingConfig) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface PublishedEmbeddingJobResult {
|
||||
metadata: RepoMeta;
|
||||
alreadyComplete: boolean;
|
||||
phase: 'embedding' | 'indexing' | 'ready';
|
||||
state?: EmbeddingJobState;
|
||||
peakQueueBytes: number;
|
||||
}
|
||||
|
||||
const digest = (value: unknown): string =>
|
||||
createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
|
||||
async function existsOwnedFile(file: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(file);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Embedding artifact must be an owned regular file');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function jobDirectory(root: string, identity: string): Promise<string> {
|
||||
const directory = await embeddingJobDirectory(root);
|
||||
return createEmbeddingJobDirectories(directory, ['embeddings', identity]);
|
||||
}
|
||||
|
||||
/** A preflight interrupted before any native/job artifact owns no paid work.
|
||||
* Only its known draft selection files may be removed. Corrupt paid receipts
|
||||
* remain evidence and must never turn into a fresh, apparently empty job.
|
||||
*/
|
||||
async function clearUnstartedManifest(directory: string): Promise<void> {
|
||||
if (
|
||||
(await existsOwnedFile(path.join(directory, 'vectors'))) ||
|
||||
(await existsOwnedFile(path.join(directory, 'job.json')))
|
||||
)
|
||||
throw new Error('Embedding manifest is missing for existing paid work; artifacts retained');
|
||||
const entries = await fs.opendir(directory);
|
||||
for await (const entry of entries) {
|
||||
if (/^(manifest\.pending|manifest\.jsonl|keys-\d+\.jsonl)$/.test(entry.name)) {
|
||||
const file = path.join(directory, entry.name);
|
||||
await existsOwnedFile(file);
|
||||
await fs.unlink(file);
|
||||
}
|
||||
}
|
||||
await syncEmbeddingJobDirectory(directory);
|
||||
}
|
||||
|
||||
export async function runPublishedEmbeddingJob(
|
||||
options: PublishedEmbeddingJobOptions,
|
||||
dependencies: PublishedEmbeddingJobDependencies = {},
|
||||
): Promise<PublishedEmbeddingJobResult> {
|
||||
const { resources, signal } = options;
|
||||
signal?.throwIfAborted();
|
||||
// A previous failed close must finish before a model or another native pool
|
||||
// can consume the next invocation's reservation.
|
||||
await drainAnalysisNativeResources();
|
||||
signal?.throwIfAborted();
|
||||
const root = await embeddingJobDirectory(options.metadataDirectory);
|
||||
// Canonicalize the parent just as we do for the job root. macOS temporary
|
||||
// paths can name the same directory through /var and /private/var.
|
||||
const graphPath = path.join(
|
||||
await embeddingJobDirectory(path.dirname(path.resolve(options.graphPath))),
|
||||
path.basename(options.graphPath),
|
||||
);
|
||||
let metadata = await options.metadata.loadMetadata();
|
||||
const graph = metadata?.graphGeneration;
|
||||
const runner = metadata?.runnerIdentity;
|
||||
if (
|
||||
!metadata ||
|
||||
!graph ||
|
||||
graph.version !== 1 ||
|
||||
!/^[a-f0-9]{32}$/.test(graph.id) ||
|
||||
!/^[a-f0-9]{64}$/.test(graph.coveredContent) ||
|
||||
!runner ||
|
||||
runner.schemaVersion !== 4 ||
|
||||
!metadata.parseCacheIdentity ||
|
||||
!metadata.schemaFingerprint ||
|
||||
metadata.incrementalInProgress ||
|
||||
metadata.graphWriteCollapsed ||
|
||||
!metadataMatchesGraph(root, metadata, graphPath)
|
||||
)
|
||||
throw new Error('Embedding job requires a verified published graph generation');
|
||||
const config = resolveEmbeddingConfig(options.config);
|
||||
const resolvedVector = (dependencies.identity ?? resolveEmbeddingIdentity)();
|
||||
const vector =
|
||||
dependencies.identity || resolvedVector.provider !== 'local'
|
||||
? resolvedVector
|
||||
: { ...resolvedVector, model: config.modelId };
|
||||
if (config.threads > resources.nativeThreads)
|
||||
throw new Error('Embedding threads exceed the admitted resource budget');
|
||||
for (const value of [resources.graphPoolBytes, resources.vectorPoolBytes, resources.queueBytes])
|
||||
if (!Number.isSafeInteger(value) || value < 8 * 1024 * 1024)
|
||||
throw new Error('Invalid embedding resource reservation');
|
||||
if (!Number.isSafeInteger(resources.nativeThreads) || resources.nativeThreads < 1)
|
||||
throw new Error('Invalid embedding CPU reservation');
|
||||
const usesLocalRuntime = vector.provider === 'local' && !dependencies.provider;
|
||||
if (
|
||||
usesLocalRuntime &&
|
||||
(!Number.isSafeInteger(resources.modelBytes) ||
|
||||
!Number.isSafeInteger(resources.modelHeapMb) ||
|
||||
resources.modelHeapMb < 128 ||
|
||||
resources.modelHeapMb * 1024 * 1024 + 48 * 1024 * 1024 >= resources.modelBytes)
|
||||
)
|
||||
throw new Error('Local embedding runtime does not fit its resource reservation');
|
||||
const identities = embeddingJobIdentity(
|
||||
{
|
||||
generation: graph.id,
|
||||
coveredContent: graph.coveredContent,
|
||||
revision: metadata.lastCommit || 'non-git',
|
||||
// Entrypoint paths are diagnostic. Identical artifacts used by analyze,
|
||||
// sync and the server must resolve to the same paid job identity.
|
||||
analyzer: digest([
|
||||
runner.schemaVersion,
|
||||
runner.runtime.version,
|
||||
runner.runtime.platform,
|
||||
runner.runtime.architecture,
|
||||
runner.runtime.modulesAbi,
|
||||
runner.runtime.libc,
|
||||
runner.cliVersion,
|
||||
runner.build.digest,
|
||||
runner.dependencyRuntime.digest,
|
||||
]),
|
||||
parser: digest(metadata.parseCacheIdentity),
|
||||
schema: metadata.schemaFingerprint,
|
||||
vectorSchema: VECTOR_STORE_SCHEMA,
|
||||
},
|
||||
vector,
|
||||
EMBEDDING_TEXT_VERSION,
|
||||
config,
|
||||
contentRetentionFromMeta(metadata),
|
||||
);
|
||||
const previous = metadata.vectorGeneration;
|
||||
if (previous?.identity === identities.identity && !metadata.embeddingJob) {
|
||||
await assertOwnedVectorGeneration(root, previous.artifactIdentity ?? previous.identity);
|
||||
return { metadata, alreadyComplete: true, phase: 'ready', peakQueueBytes: 0 };
|
||||
}
|
||||
if (metadata.embeddingJob && metadata.embeddingJob.identity !== identities.identity) {
|
||||
options.log?.(
|
||||
metadata.embeddingJob.graphGeneration !== graph.id
|
||||
? 'Embedding graph generation changed; prior paid artifacts retained for compatible reuse.'
|
||||
: 'Embedding configuration changed; prior paid artifacts retained.',
|
||||
);
|
||||
} else if (previous && previous.vectorIdentity !== identities.vectorIdentity) {
|
||||
options.log?.(
|
||||
'Embedding provider, model or dimensions changed; starting a separate vector space.',
|
||||
);
|
||||
} else if (previous && previous.inputIdentity !== identities.inputIdentity) {
|
||||
options.log?.('Embedding text options changed; prior vectors are not compatible.');
|
||||
}
|
||||
const previousCompatible =
|
||||
previous &&
|
||||
previous.identity !== identities.identity &&
|
||||
previous.vectorIdentity === identities.vectorIdentity &&
|
||||
previous.inputIdentity === identities.inputIdentity &&
|
||||
previous.dimensions === vector.dimensions;
|
||||
const limits = (hasFallback: boolean) => {
|
||||
// Writer and the sole reuse reader share this reservation. A sealed probe
|
||||
// is physically closed before its replacement writer is opened.
|
||||
const result = {
|
||||
bufferPoolBytes: Math.floor(resources.vectorPoolBytes / (hasFallback ? 2 : 1)),
|
||||
threads: resources.nativeThreads,
|
||||
batchBytes: Math.min(resources.queueBytes, 8 * 1024 * 1024),
|
||||
};
|
||||
if (result.bufferPoolBytes < 8 * 1024 * 1024 || result.batchBytes < vector.dimensions * 16)
|
||||
throw new Error('Vector writer and reuse reader do not fit their resource reservation');
|
||||
return result;
|
||||
};
|
||||
const space = {
|
||||
vectorIdentity: identities.vectorIdentity,
|
||||
inputIdentity: identities.inputIdentity,
|
||||
dimensions: vector.dimensions,
|
||||
schema: VECTOR_STORE_SCHEMA,
|
||||
};
|
||||
const validateOwner = async (owner: EmbeddingArtifactOwner) => {
|
||||
const directory = await embeddingJobDirectory(path.join(root, 'embeddings', owner.jobIdentity));
|
||||
const manifest = await loadEmbeddingWorkManifest(directory, owner.sourceIdentity);
|
||||
if (manifest.digest !== owner.manifestDigest)
|
||||
throw new Error('Owned embedding artifact manifest changed; paid artifacts retained');
|
||||
};
|
||||
let artifact = await readEmbeddingArtifactSpace(root, space, validateOwner);
|
||||
// A direct legacy receipt survives graph-bound journal replacement. Prefer
|
||||
// the newer complete artifact when one exists; never open two reuse readers.
|
||||
const legacyIdentity =
|
||||
artifact?.owner.fallbackLegacy ??
|
||||
(!artifact && !previousCompatible ? metadata.legacyVectorSource : undefined);
|
||||
let legacy = legacyIdentity ? await readLegacyEmbeddingSource(root, legacyIdentity) : undefined;
|
||||
if (legacy) {
|
||||
const incompatible =
|
||||
legacyVectorCompatibility(legacy.recordedIdentity, vector) ??
|
||||
(legacy.recordedInputIdentity !== undefined &&
|
||||
legacy.recordedInputIdentity !== identities.inputIdentity
|
||||
? 'Legacy embedding input provenance differs from this request; source retained.'
|
||||
: undefined);
|
||||
if (incompatible) {
|
||||
if (artifact?.owner.fallbackLegacy) throw new Error(incompatible);
|
||||
options.log?.(incompatible);
|
||||
legacy = undefined;
|
||||
} else {
|
||||
const current = await fs.lstat(graphPath, { bigint: true });
|
||||
if (String(current.dev) === legacy.file.dev && String(current.ino) === legacy.file.ino)
|
||||
throw new Error('Legacy reuse requires a retired source distinct from the mutable graph');
|
||||
}
|
||||
}
|
||||
if (
|
||||
!artifact &&
|
||||
metadata.embeddingJob &&
|
||||
!metadata.embeddingJob.spaceKey &&
|
||||
metadata.embeddingJob.identity !== identities.identity
|
||||
)
|
||||
options.log?.(
|
||||
'Prior pending job has no verified vector/input ownership receipt; retaining its files without assuming compatibility.',
|
||||
);
|
||||
const needsFallback = () =>
|
||||
artifact
|
||||
? artifact.owner.phase === 'sealed'
|
||||
? artifact.owner.jobIdentity !== identities.identity
|
||||
: Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy)
|
||||
: Boolean(
|
||||
legacy ||
|
||||
(previousCompatible &&
|
||||
(previous.artifactIdentity ?? previous.identity) !== identities.identity),
|
||||
);
|
||||
limits(needsFallback());
|
||||
const directory = await jobDirectory(root, identities.identity);
|
||||
let source: GraphSource | undefined;
|
||||
let store: VectorStore | undefined;
|
||||
let reuse: VectorStore | LegacyVectorSource | undefined;
|
||||
let runtime: EmbeddingSidecarReservation | undefined;
|
||||
try {
|
||||
if (usesLocalRuntime) {
|
||||
const { reserveEmbeddingSidecar } = await import('./embedding-sidecar-client.js');
|
||||
runtime = await reserveEmbeddingSidecar(config, resources.modelHeapMb);
|
||||
} else if (!dependencies.provider) {
|
||||
const { drainEmbeddingSidecarForAnalysis } = await import('./embedding-sidecar-client.js');
|
||||
await drainEmbeddingSidecarForAnalysis();
|
||||
}
|
||||
// A crash after native sealing but before recording that fact must not
|
||||
// turn the complete artifact into a mutable writer for a different graph.
|
||||
if (artifact?.owner.phase === 'pending' && artifact.owner.jobIdentity !== identities.identity) {
|
||||
const ownedDirectory = await jobDirectory(root, artifact.owner.artifactIdentity);
|
||||
if (await existsOwnedFile(path.join(ownedDirectory, 'vectors'))) {
|
||||
store = await VectorStore.open({
|
||||
directory: ownedDirectory,
|
||||
identity: artifact.owner.artifactIdentity,
|
||||
dimensions: vector.dimensions,
|
||||
mode: 'resume',
|
||||
recoverEmptyBootstrap: !(await existsOwnedFile(
|
||||
path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json'),
|
||||
)),
|
||||
...limits(Boolean(artifact.owner.fallbackArtifact || artifact.owner.fallbackLegacy)),
|
||||
});
|
||||
if (store.isSealed) {
|
||||
await store.close();
|
||||
store = undefined;
|
||||
artifact = await adoptEmbeddingArtifact(
|
||||
root,
|
||||
space,
|
||||
artifact,
|
||||
{
|
||||
...artifact.owner,
|
||||
phase: 'sealed',
|
||||
fallbackArtifact: undefined,
|
||||
fallbackLegacy: undefined,
|
||||
},
|
||||
validateOwner,
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
await existsOwnedFile(path.join(root, 'embeddings', artifact.owner.jobIdentity, 'job.json'))
|
||||
) {
|
||||
throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained');
|
||||
}
|
||||
}
|
||||
// Sealing may have been discovered by the probe. Admit both pools before
|
||||
// reading source or creating a replacement manifest.
|
||||
limits(needsFallback());
|
||||
const manifestExists = await existsOwnedFile(path.join(directory, 'manifest.json'));
|
||||
if (!manifestExists && artifact?.owner.jobIdentity === identities.identity)
|
||||
throw new Error('Owned embedding manifest is missing; paid artifacts retained');
|
||||
if (!manifestExists) await clearUnstartedManifest(directory);
|
||||
const graphElements = (metadata.stats?.nodes ?? NaN) + (metadata.stats?.edges ?? NaN);
|
||||
const sourcePoolBytes =
|
||||
Number.isSafeInteger(graphElements) && graphElements >= 0
|
||||
? Math.min(resources.graphPoolBytes, estimateBufferPool(graphElements))
|
||||
: resources.graphPoolBytes;
|
||||
const getSource = async (): Promise<GraphSource> =>
|
||||
(source ??= await (dependencies.openSource ?? EmbeddingGraphSource.open)(
|
||||
graphPath,
|
||||
sourcePoolBytes,
|
||||
resources.nativeThreads,
|
||||
));
|
||||
const manifest = manifestExists
|
||||
? await loadEmbeddingWorkManifest(directory, identities.sourceIdentity)
|
||||
: await createEmbeddingWorkManifest(directory, identities.sourceIdentity, await getSource());
|
||||
const makeOwner = (
|
||||
artifactIdentity: string,
|
||||
revalidateInputs: boolean,
|
||||
fallbackArtifact?: string,
|
||||
fallbackLegacy?: string,
|
||||
): EmbeddingArtifactOwner => ({
|
||||
artifactIdentity,
|
||||
jobIdentity: identities.identity,
|
||||
graphGeneration: graph.id,
|
||||
sourceIdentity: identities.sourceIdentity,
|
||||
manifestDigest: manifest.digest,
|
||||
phase: 'pending',
|
||||
revalidateInputs,
|
||||
...(fallbackArtifact === undefined ? {} : { fallbackArtifact }),
|
||||
...(fallbackLegacy === undefined ? {} : { fallbackLegacy }),
|
||||
});
|
||||
let owner: EmbeddingArtifactOwner;
|
||||
if (artifact?.owner.phase === 'sealed' && artifact.owner.jobIdentity === identities.identity) {
|
||||
// Publication/index-only recovery for this exact manifest. Never reopen
|
||||
// a sealed artifact with a native writer handle.
|
||||
owner = artifact.owner;
|
||||
} else if (artifact?.owner.phase === 'sealed') {
|
||||
owner = makeOwner(randomBytes(32).toString('hex'), false, artifact.owner.artifactIdentity);
|
||||
} else if (artifact) {
|
||||
owner = makeOwner(
|
||||
artifact.owner.artifactIdentity,
|
||||
artifact.owner.revalidateInputs || artifact.owner.jobIdentity !== identities.identity,
|
||||
artifact.owner.fallbackArtifact,
|
||||
artifact.owner.fallbackLegacy,
|
||||
);
|
||||
} else {
|
||||
// Start a space beside its first job. This also adopts an older bounded
|
||||
// artifact only for this exact compatible job; a digest cannot tell us
|
||||
// the vector/text provenance of a different old pending job.
|
||||
const legacyStore = await existsOwnedFile(path.join(directory, 'vectors'));
|
||||
owner = makeOwner(
|
||||
identities.identity,
|
||||
false,
|
||||
previousCompatible ? (previous.artifactIdentity ?? previous.identity) : undefined,
|
||||
legacy?.identity,
|
||||
);
|
||||
if (owner.fallbackArtifact === owner.artifactIdentity) owner.fallbackArtifact = undefined;
|
||||
if (
|
||||
legacyStore &&
|
||||
previous?.identity === identities.identity &&
|
||||
(previous.artifactIdentity ?? previous.identity) === owner.artifactIdentity
|
||||
)
|
||||
owner.phase = 'sealed';
|
||||
}
|
||||
artifact = await adoptEmbeddingArtifact(root, space, artifact, owner, validateOwner);
|
||||
const storeResources = limits(Boolean(owner.fallbackArtifact || owner.fallbackLegacy));
|
||||
if (
|
||||
metadata.embeddingJob?.identity !== identities.identity ||
|
||||
metadata.embeddingJob.artifactIdentity !== owner.artifactIdentity ||
|
||||
metadata.embeddingJob.spaceKey !== artifact.spaceKey
|
||||
) {
|
||||
metadata = {
|
||||
...metadata,
|
||||
embeddingJob: {
|
||||
version: 1,
|
||||
identity: identities.identity,
|
||||
graphGeneration: graph.id,
|
||||
artifactIdentity: owner.artifactIdentity,
|
||||
spaceKey: artifact.spaceKey,
|
||||
},
|
||||
};
|
||||
await options.metadata.saveMetadata(metadata);
|
||||
await syncPublishedMetadata(root, metadata);
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
const artifactDirectory = await jobDirectory(root, owner.artifactIdentity);
|
||||
if (owner.artifactIdentity !== owner.jobIdentity) {
|
||||
// Rebased artifacts live apart from the journal. Keep independent proof
|
||||
// of ownership so a later generation can reclaim their native payload.
|
||||
try {
|
||||
const receipt = (await readEmbeddingJobRecord(artifactDirectory, 'artifact.json')) as {
|
||||
version: number;
|
||||
identity: string;
|
||||
jobIdentity: string;
|
||||
};
|
||||
if (
|
||||
receipt.version !== 1 ||
|
||||
receipt.identity !== owner.artifactIdentity ||
|
||||
!/^[a-f0-9]{64}$/.test(receipt.jobIdentity)
|
||||
)
|
||||
throw new Error('Embedding artifact has unknown ownership');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
await writeEmbeddingJobRecord(artifactDirectory, 'artifact.json', {
|
||||
version: 1,
|
||||
identity: owner.artifactIdentity,
|
||||
jobIdentity: owner.jobIdentity,
|
||||
});
|
||||
}
|
||||
}
|
||||
const hasStore = await existsOwnedFile(path.join(artifactDirectory, 'vectors'));
|
||||
const hasJournal = await existsOwnedFile(path.join(directory, 'job.json'));
|
||||
if (!hasStore && hasJournal)
|
||||
throw new Error('Embedding vectors are missing for acknowledged work; artifacts retained');
|
||||
store ??= await VectorStore.open({
|
||||
directory: artifactDirectory,
|
||||
identity: owner.artifactIdentity,
|
||||
dimensions: vector.dimensions,
|
||||
mode: owner.phase === 'sealed' ? 'read' : hasStore ? 'resume' : 'create',
|
||||
recoverEmptyBootstrap: !hasJournal,
|
||||
...storeResources,
|
||||
});
|
||||
const journal = await EmbeddingJobJournal.recover(directory, identities.identity, manifest);
|
||||
if (owner.fallbackArtifact) {
|
||||
const file = await assertOwnedVectorGeneration(root, owner.fallbackArtifact);
|
||||
reuse = await VectorStore.open({
|
||||
directory: path.dirname(file),
|
||||
identity: owner.fallbackArtifact,
|
||||
dimensions: vector.dimensions,
|
||||
mode: 'read',
|
||||
...storeResources,
|
||||
});
|
||||
} else if (owner.fallbackLegacy && !store.isSealed) {
|
||||
if (!legacy || legacy.identity !== owner.fallbackLegacy)
|
||||
throw new Error('Legacy source does not match the owned fallback; artifacts retained');
|
||||
reuse = await LegacyVectorSource.open(
|
||||
{
|
||||
path: legacy.path,
|
||||
sourceIdentity: legacy.identity,
|
||||
recordedIdentity: legacy.recordedIdentity,
|
||||
expectedIdentity: vector,
|
||||
recordedInputIdentity: legacy.recordedInputIdentity,
|
||||
expectedInputIdentity: identities.inputIdentity,
|
||||
config,
|
||||
signal,
|
||||
log: options.log,
|
||||
...storeResources,
|
||||
},
|
||||
store,
|
||||
legacy.excludedIds,
|
||||
);
|
||||
}
|
||||
const provider: typeof embedBatch =
|
||||
dependencies.provider ??
|
||||
(async (texts, request) => {
|
||||
if (runtime) return runtime.embedBatch(texts, request);
|
||||
const { embedBatch } = await import('./embedder.js');
|
||||
return embedBatch(texts, {
|
||||
...request,
|
||||
maxResponseBytes: texts.length * (4096 + 32 * vector.dimensions),
|
||||
});
|
||||
});
|
||||
const initializeProvider = async () => {
|
||||
if (dependencies.initializeProvider) return dependencies.initializeProvider(config);
|
||||
await runtime?.initialize(options.onModelProgress);
|
||||
};
|
||||
const result = await runEmbeddingJob({
|
||||
directory,
|
||||
manifest,
|
||||
journal,
|
||||
store,
|
||||
reuse,
|
||||
// A sealed artifact still belongs to this exact job/manifest; rebuilding
|
||||
// a damaged cursor can use its complete groups without source reads.
|
||||
revalidateStoredInputs: owner.revalidateInputs && !store.isSealed,
|
||||
config,
|
||||
queueBytes: resources.queueBytes,
|
||||
sourceContext: {
|
||||
repoPath: metadata.repoPath,
|
||||
fileHashes: metadata.fileHashes,
|
||||
contentRetention: contentRetentionFromMeta(metadata),
|
||||
},
|
||||
// Opening the graph is lazy even on resume: indexing-only retries do not
|
||||
// enumerate/fetch source or initialize a provider.
|
||||
source: {
|
||||
scan: async function* (label) {
|
||||
yield* (await getSource()).scan(label);
|
||||
},
|
||||
fetch: async (label, ids) => (await getSource()).fetch(label, ids),
|
||||
},
|
||||
provider,
|
||||
initializeProvider,
|
||||
signal,
|
||||
onProgress: options.onProgress,
|
||||
});
|
||||
const sealed = store.isSealed;
|
||||
await store.close();
|
||||
store = undefined;
|
||||
await reuse?.close();
|
||||
reuse = undefined;
|
||||
await source?.close();
|
||||
source = undefined;
|
||||
signal?.throwIfAborted();
|
||||
if (!result.indexKind)
|
||||
return { metadata, alreadyComplete: false, phase: 'embedding', ...result };
|
||||
if (sealed && artifact.owner.phase !== 'sealed') {
|
||||
artifact = await adoptEmbeddingArtifact(
|
||||
root,
|
||||
space,
|
||||
artifact,
|
||||
{
|
||||
...artifact.owner,
|
||||
phase: 'sealed',
|
||||
fallbackArtifact: undefined,
|
||||
fallbackLegacy: undefined,
|
||||
},
|
||||
validateOwner,
|
||||
);
|
||||
}
|
||||
const published = await publishVectorGeneration(
|
||||
root,
|
||||
{
|
||||
version: 1,
|
||||
identity: identities.identity,
|
||||
artifactIdentity: owner.artifactIdentity,
|
||||
graphGeneration: graph.id,
|
||||
vectorIdentity: identities.vectorIdentity,
|
||||
inputIdentity: identities.inputIdentity,
|
||||
dimensions: vector.dimensions,
|
||||
nodes: manifest.totalNodes,
|
||||
chunks: result.state.chunks,
|
||||
...(result.state.excludedNodes
|
||||
? {
|
||||
excludedNodes: result.state.excludedNodes,
|
||||
exclusionReason: 'input-rejected' as const,
|
||||
}
|
||||
: {}),
|
||||
indexKind: result.indexKind,
|
||||
},
|
||||
storeResources,
|
||||
options.metadata,
|
||||
legacy?.identity,
|
||||
);
|
||||
if (!published) return { metadata, alreadyComplete: false, phase: 'indexing', ...result };
|
||||
if (result.state.excludedNodes)
|
||||
options.log?.(
|
||||
`Skipped ${result.state.excludedNodes} node(s) after three deterministic embedding input rejections; compatible vectors were published.`,
|
||||
);
|
||||
if (journal.snapshot().phase !== 'ready') await journal.ready();
|
||||
options.onProgress?.(journal.snapshot(), result.peakQueueBytes);
|
||||
return {
|
||||
metadata: published,
|
||||
alreadyComplete: false,
|
||||
phase: 'ready',
|
||||
state: journal.snapshot(),
|
||||
peakQueueBytes: result.peakQueueBytes,
|
||||
};
|
||||
} finally {
|
||||
// Sequential finally blocks close every native handle even when one close fails.
|
||||
try {
|
||||
await store?.close();
|
||||
} finally {
|
||||
try {
|
||||
await reuse?.close();
|
||||
} finally {
|
||||
try {
|
||||
await source?.close();
|
||||
} finally {
|
||||
runtime?.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -65,6 +65,7 @@ import {
|
|||
needsEmbeddingSource,
|
||||
type EmbeddingSourceContext,
|
||||
} from './source-reader.js';
|
||||
import type { VectorSearchSnapshot } from './vector-generation-search.js';
|
||||
|
||||
const isDev = process.env.NODE_ENV === 'development';
|
||||
|
||||
|
|
@ -115,7 +116,6 @@ export const contentHashForNode = (
|
|||
): string => {
|
||||
// Hash must be deterministic across runs, so exclude methodNames/fieldNames
|
||||
// which are populated during the batch loop via AST extraction.
|
||||
// Using only node.content ensures the hash stays stable.
|
||||
// NOTE: A change to extractStructuralNames behavior requires bumping EMBEDDING_TEXT_VERSION.
|
||||
const hashNode = { ...node, methodNames: undefined, fieldNames: undefined };
|
||||
if (node.label === 'Class') hashNode.classDeclaration = undefined;
|
||||
|
|
@ -181,7 +181,7 @@ export const contentHashForNode = (
|
|||
return hash.digest('hex');
|
||||
};
|
||||
|
||||
const isInputRejection = (error: unknown): boolean =>
|
||||
export const isInputRejection = (error: unknown): boolean =>
|
||||
error instanceof HttpEmbeddingError &&
|
||||
(error.status === 400 || error.status === 413 || error.status === 422);
|
||||
|
||||
|
|
@ -193,14 +193,15 @@ type EmbeddedRange = { start: number; end: number } & (
|
|||
/** Bisect only input rejections, with at most 2n-1 requests per sub-batch.
|
||||
* Authentication, throttling, transport and cancellation retain their existing
|
||||
* bounded retry/failure policy instead of multiplying requests. */
|
||||
async function* embedInputRanges(
|
||||
export async function* embedInputRanges(
|
||||
texts: string[],
|
||||
subBatchSize: number,
|
||||
signal?: AbortSignal,
|
||||
provider: typeof embedBatch = embedBatch,
|
||||
): AsyncGenerator<EmbeddedRange> {
|
||||
async function* request(start: number, end: number): AsyncGenerator<EmbeddedRange> {
|
||||
try {
|
||||
const embeddings = await embedBatch(texts.slice(start, end), { signal });
|
||||
const embeddings = await provider(texts.slice(start, end), { signal });
|
||||
yield { start, end, embeddings };
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isCancellationError(error)) throw error;
|
||||
|
|
@ -1280,34 +1281,45 @@ export const semanticSearch = async (
|
|||
query: string,
|
||||
k: number = 10,
|
||||
maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE),
|
||||
snapshot?: VectorSearchSnapshot,
|
||||
embedQuery?: (text: string) => Promise<readonly number[]>,
|
||||
): Promise<SemanticSearchResult[]> => {
|
||||
// determinism: probe — existence only. Only `exists.length` is read; which
|
||||
// row LIMIT 1 returns cannot change whether the table is empty.
|
||||
const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`);
|
||||
if (!exists.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!isEmbedderReady()) {
|
||||
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
|
||||
}
|
||||
|
||||
const queryEmbedding = await embedText(query);
|
||||
const queryVec = embeddingToArray(queryEmbedding);
|
||||
const queryVecStr = `[${queryVec.join(',')}]`;
|
||||
|
||||
const queryVector = async (): Promise<number[]> => {
|
||||
if (embedQuery) return [...(await embedQuery(query))];
|
||||
if (!isEmbedderReady())
|
||||
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
|
||||
return embeddingToArray(await embedText(query));
|
||||
};
|
||||
let bestChunks = new Map<
|
||||
string,
|
||||
{ distance: number; chunkIndex: number; startLine: number; endLine: number }
|
||||
>();
|
||||
// Query/read path: NEVER spawn a network INSTALL on a user query. If the
|
||||
// VECTOR extension was not pre-installed, fall back to exact scan rather than
|
||||
// blocking the query on a download (offline-first; see extension-loader.ts
|
||||
// "load-only" — used by all serve/MCP query paths).
|
||||
if (await loadVectorExtension(undefined, { policy: 'load-only' })) {
|
||||
try {
|
||||
bestChunks = await collectBestChunks(k, async (fetchLimit) => {
|
||||
const vectorQuery = `
|
||||
let generationSelected = false;
|
||||
if (snapshot) {
|
||||
const { searchVectorGeneration } = await import('./vector-generation-search.js');
|
||||
const chunks = await searchVectorGeneration(snapshot, queryVector, k, maxDistance);
|
||||
generationSelected = chunks !== null;
|
||||
if (chunks) bestChunks = new Map(chunks.map(({ nodeId, ...chunk }) => [nodeId, chunk]));
|
||||
}
|
||||
if (!generationSelected) {
|
||||
// determinism: probe — existence only. Only `exists.length` is read; which
|
||||
// row LIMIT 1 returns cannot change whether the table is empty.
|
||||
const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`);
|
||||
if (!exists.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const queryVec = await queryVector();
|
||||
const queryVecStr = `[${queryVec.join(',')}]`;
|
||||
|
||||
// Query/read path: NEVER spawn a network INSTALL on a user query. If the
|
||||
// VECTOR extension was not pre-installed, fall back to exact scan rather than
|
||||
// blocking the query on a download (offline-first; see extension-loader.ts
|
||||
// "load-only" — used by all serve/MCP query paths).
|
||||
if (await loadVectorExtension(undefined, { policy: 'load-only' })) {
|
||||
try {
|
||||
bestChunks = await collectBestChunks(k, async (fetchLimit) => {
|
||||
const vectorQuery = `
|
||||
CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
|
||||
CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit})
|
||||
YIELD node AS emb, distance
|
||||
|
|
@ -1318,54 +1330,54 @@ export const semanticSearch = async (
|
|||
ORDER BY distance
|
||||
`;
|
||||
|
||||
const embResults = await executeQuery(vectorQuery);
|
||||
return embResults.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
distance: row.distance ?? row[4],
|
||||
}));
|
||||
});
|
||||
} catch {
|
||||
bestChunks = new Map();
|
||||
const embResults = await executeQuery(vectorQuery);
|
||||
return embResults.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
distance: row.distance ?? row[4],
|
||||
}));
|
||||
});
|
||||
} catch {
|
||||
bestChunks = new Map();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestChunks.size === 0) {
|
||||
// NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher
|
||||
// never writes a fabricated 0, whereas here `?? 0` is the right answer —
|
||||
// an unknown count simply skips the exact scan.
|
||||
const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER);
|
||||
const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0);
|
||||
const exactLimit = getExactScanLimit();
|
||||
if (embeddingCount > 0 && embeddingCount <= exactLimit) {
|
||||
const rows = await executeQuery(`
|
||||
if (bestChunks.size === 0) {
|
||||
// NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher
|
||||
// never writes a fabricated 0, whereas here `?? 0` is the right answer —
|
||||
// an unknown count simply skips the exact scan.
|
||||
const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER);
|
||||
const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0);
|
||||
const exactLimit = getExactScanLimit();
|
||||
if (embeddingCount > 0 && embeddingCount <= exactLimit) {
|
||||
const rows = await executeQuery(`
|
||||
MATCH (e:${EMBEDDING_TABLE_NAME})
|
||||
RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex,
|
||||
e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding
|
||||
`);
|
||||
const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
embedding: row.embedding ?? row[4] ?? [],
|
||||
}));
|
||||
bestChunks = new Map(
|
||||
rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [
|
||||
row.nodeId,
|
||||
{
|
||||
distance: row.distance,
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
},
|
||||
]),
|
||||
);
|
||||
const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
embedding: row.embedding ?? row[4] ?? [],
|
||||
}));
|
||||
bestChunks = new Map(
|
||||
rankExactEmbeddingRows(exactRows, queryVec, k, maxDistance).map((row) => [
|
||||
row.nodeId,
|
||||
{
|
||||
distance: row.distance,
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestChunks.size === 0) {
|
||||
return [];
|
||||
}
|
||||
|
|
@ -1431,8 +1443,9 @@ export const semanticSearchWithContext = async (
|
|||
query: string,
|
||||
k: number = 5,
|
||||
_hops: number = 1,
|
||||
snapshot?: VectorSearchSnapshot,
|
||||
): Promise<any[]> => {
|
||||
const results = await semanticSearch(executeQuery, query, k);
|
||||
const results = await semanticSearch(executeQuery, query, k, undefined, snapshot);
|
||||
|
||||
return results.map((r) => ({
|
||||
matchId: r.nodeId,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import type {
|
|||
SidecarResponse,
|
||||
} from './embedding-sidecar-protocol.js';
|
||||
import { logger } from '../logger.js';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import { resolveEmbeddingConfig } from './config.js';
|
||||
|
||||
export type ForkImpl = (
|
||||
modulePath: string,
|
||||
|
|
@ -47,10 +49,16 @@ let lastInitOptions:
|
|||
| {
|
||||
embeddingConfig?: Partial<EmbeddingConfig>;
|
||||
forceDevice?: EmbeddingSidecarDevice;
|
||||
heapLimitMb?: number;
|
||||
}
|
||||
| undefined;
|
||||
let exitHooked = false;
|
||||
let progressSink: ((progress: ModelProgress) => void) | undefined;
|
||||
// A caller abort only stops waiting. The child may still be inside ONNX; keep
|
||||
// its physical requests and process allocation until a response/close proves release.
|
||||
const physicalRequests = new Map<number, ChildProcess>();
|
||||
const liveChildren = new Set<ChildProcess>();
|
||||
let runtimeReservation: { config: EmbeddingConfig; heapLimitMb: number } | undefined;
|
||||
|
||||
const pending = new Map<
|
||||
number,
|
||||
|
|
@ -75,6 +83,9 @@ export const _resetEmbeddingSidecarForTests = (): void => {
|
|||
device = 'cpu';
|
||||
ensureChain = null;
|
||||
lastInitOptions = undefined;
|
||||
runtimeReservation = undefined;
|
||||
physicalRequests.clear();
|
||||
liveChildren.clear();
|
||||
};
|
||||
|
||||
const sidecarScriptPath = (): string => {
|
||||
|
|
@ -91,9 +102,26 @@ const tsxHookArgs = (): string[] => {
|
|||
return ['--import', pathToFileURL(require.resolve('tsx/esm')).href];
|
||||
};
|
||||
|
||||
const childEnv = (): NodeJS.ProcessEnv => {
|
||||
const childEnv = (configPinned = false): NodeJS.ProcessEnv => {
|
||||
const env = { ...process.env };
|
||||
delete env.GITNEXUS_EMBEDDING_URL;
|
||||
if (configPinned) {
|
||||
// The admitted, resolved config travels over IPC. An environment mutation
|
||||
// between admission and lazy model init must not override its allocations.
|
||||
for (const name of [
|
||||
'THREADS',
|
||||
'DEVICE',
|
||||
'BATCH_SIZE',
|
||||
'SUB_BATCH_SIZE',
|
||||
'MAX_INPUT_CHARS',
|
||||
'MODEL_ID',
|
||||
'CHUNK_SIZE',
|
||||
'CHUNK_OVERLAP',
|
||||
'CHUNK_STRATEGY',
|
||||
]) {
|
||||
delete env[`GITNEXUS_EMBEDDING_${name}`];
|
||||
}
|
||||
}
|
||||
return env;
|
||||
};
|
||||
|
||||
|
|
@ -146,10 +174,14 @@ const rejectAll = (error: Error): void => {
|
|||
};
|
||||
|
||||
const attachChild = (proc: ChildProcess): void => {
|
||||
liveChildren.add(proc);
|
||||
proc.stderr?.on('data', (chunk: Buffer | string) => {
|
||||
logger.debug({ sidecar: true }, String(chunk).trimEnd());
|
||||
});
|
||||
proc.on('message', (msg: SidecarResponse) => {
|
||||
if (msg.type !== 'progress' && physicalRequests.get(msg.id) === proc)
|
||||
physicalRequests.delete(msg.id);
|
||||
if (child !== proc) return;
|
||||
if (msg.type === 'progress') {
|
||||
progressSink?.({
|
||||
status: msg.status,
|
||||
|
|
@ -162,6 +194,8 @@ const attachChild = (proc: ChildProcess): void => {
|
|||
waiter.resolve(msg);
|
||||
});
|
||||
proc.on('close', (code, signal) => {
|
||||
liveChildren.delete(proc);
|
||||
for (const [id, owner] of physicalRequests) if (owner === proc) physicalRequests.delete(id);
|
||||
if (child !== proc) return;
|
||||
noteChildDeath(signal);
|
||||
child = null;
|
||||
|
|
@ -183,6 +217,7 @@ const request = (
|
|||
signal?: AbortSignal,
|
||||
): Promise<SidecarResponse> => {
|
||||
if (!child) return Promise.reject(new Error('Embedding sidecar is not running'));
|
||||
const proc = child;
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
} catch (err) {
|
||||
|
|
@ -210,7 +245,7 @@ const request = (
|
|||
};
|
||||
const timer = setTimeout(() => {
|
||||
finish(() => {
|
||||
child?.kill('SIGKILL');
|
||||
proc.kill('SIGKILL');
|
||||
reject(new Error(`Embedding sidecar request timed out after ${timeoutMs}ms (${msg.type})`));
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
|
@ -225,8 +260,10 @@ const request = (
|
|||
return;
|
||||
}
|
||||
try {
|
||||
child!.send({ ...msg, id } as SidecarRequest);
|
||||
physicalRequests.set(id, proc);
|
||||
proc.send({ ...msg, id } as SidecarRequest);
|
||||
} catch (err) {
|
||||
physicalRequests.delete(id);
|
||||
finish(() => {
|
||||
reject(err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
|
|
@ -234,11 +271,19 @@ const request = (
|
|||
});
|
||||
};
|
||||
|
||||
const spawnSidecar = (): ChildProcess => {
|
||||
const spawnSidecar = (heapLimitMb?: number): ChildProcess => {
|
||||
if (liveChildren.size > 0) {
|
||||
throw new Error('Previous embedding sidecar has not released its native resource reservation');
|
||||
}
|
||||
const proc = forkImpl(sidecarScriptPath(), [], {
|
||||
execArgv: tsxHookArgs(),
|
||||
execArgv: [
|
||||
...tsxHookArgs(),
|
||||
...(heapLimitMb === undefined
|
||||
? []
|
||||
: [`--max-old-space-size=${heapLimitMb}`, '--max-semi-space-size=16']),
|
||||
],
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
env: childEnv(),
|
||||
env: childEnv(runtimeReservation !== undefined),
|
||||
});
|
||||
if (!exitHooked) {
|
||||
exitHooked = true;
|
||||
|
|
@ -263,21 +308,32 @@ export const reapEmbeddingSidecar = (): void => {
|
|||
};
|
||||
|
||||
/** Reap and wait for the killed child's close/error so an awaited dispose is a real boundary. */
|
||||
export const reapEmbeddingSidecarAndWait = async (timeoutMs = 5_000): Promise<void> => {
|
||||
export const reapEmbeddingSidecarAndWait = async (
|
||||
timeoutMs = 5_000,
|
||||
options: { requireExit?: boolean } = {},
|
||||
): Promise<void> => {
|
||||
const proc = child;
|
||||
if (!proc) return;
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
const finish = (): void => resolve();
|
||||
proc.once('close', finish);
|
||||
proc.once('error', finish);
|
||||
if (!options.requireExit) proc.once('error', finish);
|
||||
});
|
||||
reapEmbeddingSidecar();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
closed,
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, timeoutMs);
|
||||
new Promise<void>((resolve, reject) => {
|
||||
timer = setTimeout(
|
||||
() =>
|
||||
options.requireExit
|
||||
? reject(
|
||||
new Error('Embedding sidecar has not released its native resource reservation'),
|
||||
)
|
||||
: resolve(),
|
||||
timeoutMs,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
|
|
@ -314,11 +370,15 @@ const spawnAndInit = async (options?: EnsureSidecarOptions): Promise<void> => {
|
|||
}
|
||||
|
||||
const persisted = {
|
||||
embeddingConfig: options?.embeddingConfig ?? lastInitOptions?.embeddingConfig,
|
||||
forceDevice: options?.forceDevice ?? lastInitOptions?.forceDevice,
|
||||
embeddingConfig:
|
||||
runtimeReservation?.config ??
|
||||
resolveEmbeddingConfig(options?.embeddingConfig ?? lastInitOptions?.embeddingConfig),
|
||||
forceDevice:
|
||||
options?.forceDevice ?? (runtimeReservation ? undefined : lastInitOptions?.forceDevice),
|
||||
heapLimitMb: runtimeReservation?.heapLimitMb ?? lastInitOptions?.heapLimitMb,
|
||||
};
|
||||
|
||||
child = spawnSidecar();
|
||||
child = spawnSidecar(persisted.heapLimitMb);
|
||||
attachChild(child);
|
||||
progressSink = options?.onProgress;
|
||||
try {
|
||||
|
|
@ -386,6 +446,26 @@ const raceAbort = async <T>(promise: Promise<T>, signal?: AbortSignal): Promise<
|
|||
export const ensureEmbeddingSidecar = async (
|
||||
options?: EnsureSidecarOptions,
|
||||
): Promise<{ device: EmbeddingSidecarDevice }> => {
|
||||
if (runtimeReservation) {
|
||||
if (
|
||||
options?.forceDevice &&
|
||||
runtimeReservation.config.device !== 'auto' &&
|
||||
options.forceDevice !== runtimeReservation.config.device
|
||||
) {
|
||||
throw new Error('Embedding device conflicts with an active job resource reservation');
|
||||
}
|
||||
for (const [key, value] of Object.entries(options?.embeddingConfig ?? {})) {
|
||||
if (
|
||||
value !== undefined &&
|
||||
value !== runtimeReservation.config[key as keyof EmbeddingConfig]
|
||||
) {
|
||||
throw new Error(
|
||||
'Embedding configuration conflicts with an active job resource reservation',
|
||||
);
|
||||
}
|
||||
}
|
||||
options = { ...options, embeddingConfig: runtimeReservation.config };
|
||||
}
|
||||
if (localUnavailable) throw localUnavailableError();
|
||||
if (ready && child) {
|
||||
rejectConflictingForceDevice(options?.forceDevice);
|
||||
|
|
@ -403,6 +483,69 @@ export const ensureEmbeddingSidecar = async (
|
|||
return { device };
|
||||
};
|
||||
|
||||
export interface EmbeddingSidecarReservation {
|
||||
initialize(onProgress?: (progress: ModelProgress) => void): Promise<void>;
|
||||
embedBatch(texts: string[], options?: { signal?: AbortSignal }): Promise<Float32Array[]>;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/** Admit before opening job databases; acquire does not load a model. A warm
|
||||
* compatible sidecar remains usable, and only one job can reconfigure it. */
|
||||
export async function reserveEmbeddingSidecar(
|
||||
config: EmbeddingConfig,
|
||||
heapLimitMb: number,
|
||||
): Promise<EmbeddingSidecarReservation> {
|
||||
if (!Number.isSafeInteger(heapLimitMb) || heapLimitMb < 128)
|
||||
throw new Error('Embedding sidecar heap allocation must be at least 128 MiB');
|
||||
if (runtimeReservation || ensureChain || physicalRequests.size > 0)
|
||||
throw new Error('Embedding runtime is busy; cannot reassign its resource reservation');
|
||||
const reservation = { config: Object.freeze({ ...config }), heapLimitMb };
|
||||
runtimeReservation = reservation;
|
||||
try {
|
||||
if (
|
||||
child &&
|
||||
(!ready ||
|
||||
lastInitOptions?.heapLimitMb !== heapLimitMb ||
|
||||
!isDeepStrictEqual(lastInitOptions?.embeddingConfig, config))
|
||||
) {
|
||||
await reapEmbeddingSidecarAndWait(5_000, { requireExit: true });
|
||||
}
|
||||
if (liveChildren.size > (child ? 1 : 0))
|
||||
throw new Error(
|
||||
'Previous embedding sidecar has not released its native resource reservation',
|
||||
);
|
||||
const assertOwned = () => {
|
||||
if (runtimeReservation !== reservation)
|
||||
throw new Error('Embedding runtime reservation was released');
|
||||
};
|
||||
return {
|
||||
initialize: async (onProgress) => {
|
||||
assertOwned();
|
||||
await ensureEmbeddingSidecar({ onProgress, embeddingConfig: reservation.config });
|
||||
},
|
||||
embedBatch: async (texts, options) => {
|
||||
assertOwned();
|
||||
return sidecarEmbedBatch(texts, options);
|
||||
},
|
||||
release: () => {
|
||||
if (runtimeReservation === reservation) runtimeReservation = undefined;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (runtimeReservation === reservation) runtimeReservation = undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parser phases reuse the model reservation only after an idle child exits. */
|
||||
export async function drainEmbeddingSidecarForAnalysis(): Promise<void> {
|
||||
if (runtimeReservation || ensureChain || physicalRequests.size > 0)
|
||||
throw new Error('Embedding runtime is busy; refusing to reuse its native resource reservation');
|
||||
await reapEmbeddingSidecarAndWait(5_000, { requireExit: true });
|
||||
if (liveChildren.size > 0)
|
||||
throw new Error('Previous embedding sidecar has not released its native resource reservation');
|
||||
}
|
||||
|
||||
const vectorsFromEmbedResponse = (response: SidecarResponse): Float32Array[] => {
|
||||
if (response.type === 'error') throw new Error(response.message);
|
||||
if (response.type !== 'vectors') {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ process.on('message', (msg: SidecarRequest) => {
|
|||
void handle(msg);
|
||||
});
|
||||
|
||||
// This is a standalone process, not a worker thread. If its owner exits or is
|
||||
// killed after cancellation grace, release the model instead of orphaning it.
|
||||
process.once('disconnect', () => process.exit(1));
|
||||
|
||||
async function handle(msg: SidecarRequest): Promise<void> {
|
||||
try {
|
||||
switch (msg.type) {
|
||||
|
|
|
|||
385
gitnexus/src/core/embeddings/embedding-work-manifest.ts
Normal file
385
gitnexus/src/core/embeddings/embedding-work-manifest.ts
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import fs, { type FileHandle } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { assertValidGraphNodeId } from '../graph/node-identity.js';
|
||||
import { EmbeddingByteBudget } from '../analysis-resource-budget.js';
|
||||
import { EMBEDDABLE_LABELS, type EmbeddableNode } from './types.js';
|
||||
import {
|
||||
embeddingJobDirectory,
|
||||
openEmbeddingJobFile,
|
||||
readEmbeddingJobRecord,
|
||||
syncEmbeddingJobDirectory,
|
||||
writeEmbeddingJobRecord,
|
||||
} from '../../storage/embedding-job.js';
|
||||
|
||||
// Persisted unit identity must not depend on a retry's memory/batch settings.
|
||||
const UNIT_NODES = 64;
|
||||
const UNIT_SOURCE_BYTES = 1024 * 1024;
|
||||
const MAX_LINE_BYTES = 1024 * 1024;
|
||||
const IO_BYTES = 64 * 1024;
|
||||
const LABELS = new Set<string>([...EMBEDDABLE_LABELS, 'File']);
|
||||
|
||||
export interface EmbeddingWorkKey {
|
||||
id: string;
|
||||
contentBytes: number;
|
||||
/** Upper bound for serialized persisted node metadata, including its ID. */
|
||||
metadataBytes: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingManifestSource {
|
||||
/** Key-only scan in UTF-8 ID order. File scans yield only eligible text files. */
|
||||
scan(label: string): AsyncIterable<EmbeddingWorkKey>;
|
||||
/** Bounded primary-key fetch against the same immutable graph generation. */
|
||||
fetch(label: string, ids: readonly string[]): Promise<EmbeddableNode[]>;
|
||||
}
|
||||
|
||||
export interface EmbeddingWorkUnit {
|
||||
ordinal: number;
|
||||
label: string;
|
||||
nodes: EmbeddingWorkKey[];
|
||||
/** Source strings plus persisted metadata; excludes later text/vector queues. */
|
||||
retainedBytes: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingManifestReceipt {
|
||||
version: 1;
|
||||
sourceIdentity: string;
|
||||
digest: string;
|
||||
bytes: number;
|
||||
totalNodes: number;
|
||||
totalUnits: number;
|
||||
maximumUnitBytes: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingManifestCursor {
|
||||
unit: number;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
function count(value: unknown): value is number {
|
||||
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
function keyBytes(key: EmbeddingWorkKey): number {
|
||||
return 2 * (key.contentBytes + key.metadataBytes);
|
||||
}
|
||||
|
||||
function validateKey(value: unknown): asserts value is EmbeddingWorkKey {
|
||||
if (!value || typeof value !== 'object' || !('id' in value))
|
||||
throw new Error('Malformed embedding manifest key');
|
||||
assertValidGraphNodeId(value.id, 'Embedding manifest');
|
||||
if (
|
||||
Buffer.from(value.id).toString('utf8') !== value.id ||
|
||||
!('contentBytes' in value) ||
|
||||
!count(value.contentBytes) ||
|
||||
!('metadataBytes' in value) ||
|
||||
!count(value.metadataBytes) ||
|
||||
value.metadataBytes < Buffer.byteLength(value.id)
|
||||
) {
|
||||
throw new Error('Invalid embedding manifest key bounds');
|
||||
}
|
||||
if (!Number.isSafeInteger(2 * (value.contentBytes + value.metadataBytes)))
|
||||
throw new Error('Embedding key size exceeds safe accounting');
|
||||
}
|
||||
|
||||
class BufferedLines {
|
||||
private pending: string[] = [];
|
||||
private size = 0;
|
||||
constructor(private readonly handle: FileHandle) {}
|
||||
async write(line: string): Promise<void> {
|
||||
const size = Buffer.byteLength(line);
|
||||
if (size > MAX_LINE_BYTES)
|
||||
throw new Error('Embedding manifest record exceeds its bounded size');
|
||||
this.pending.push(line);
|
||||
this.size += size;
|
||||
if (this.size >= IO_BYTES) await this.flush();
|
||||
}
|
||||
async flush(): Promise<void> {
|
||||
if (!this.size) return;
|
||||
await this.handle.writeFile(this.pending.join(''));
|
||||
this.pending = [];
|
||||
this.size = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Framing stays bounded even for a corrupt file without a newline. */
|
||||
async function* lines(file: string, start = 0): AsyncGenerator<{ text: string; end: number }> {
|
||||
const { handle } = await openEmbeddingJobFile(file);
|
||||
try {
|
||||
const buffer = Buffer.alloc(IO_BYTES);
|
||||
let pending = Buffer.alloc(0);
|
||||
let position = start;
|
||||
let lineStart = start;
|
||||
for (;;) {
|
||||
const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
|
||||
if (!bytesRead) break;
|
||||
position += bytesRead;
|
||||
const data = pending.length
|
||||
? Buffer.concat([pending, buffer.subarray(0, bytesRead)])
|
||||
: buffer.subarray(0, bytesRead);
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const end = data.indexOf(10, offset);
|
||||
if (end < 0) break;
|
||||
if (end - offset + 1 > MAX_LINE_BYTES)
|
||||
throw new Error('Embedding manifest record exceeds its bounded size');
|
||||
lineStart += end - offset + 1;
|
||||
yield {
|
||||
text: new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)),
|
||||
end: lineStart,
|
||||
};
|
||||
offset = end + 1;
|
||||
}
|
||||
pending = Buffer.from(data.subarray(offset));
|
||||
if (pending.length >= MAX_LINE_BYTES)
|
||||
throw new Error('Embedding manifest record exceeds its bounded size');
|
||||
}
|
||||
if (pending.length) throw new Error('Embedding manifest has an incomplete trailing record');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Merge label-key spools to detect cross-label duplicates without a corpus Set. */
|
||||
async function verifyUniqueKeys(files: string[]): Promise<void> {
|
||||
const readers = files.map((file) => lines(file));
|
||||
const heads: Array<{ id: string; bytes: Buffer } | undefined> = [];
|
||||
const head = (text: string): { id: string; bytes: Buffer } => {
|
||||
const { id }: { id: string } = JSON.parse(text);
|
||||
return { id, bytes: Buffer.from(id) };
|
||||
};
|
||||
try {
|
||||
for (const reader of readers) {
|
||||
const next = await reader.next();
|
||||
heads.push(next.done ? undefined : head(next.value.text));
|
||||
}
|
||||
let previous: string | undefined;
|
||||
for (;;) {
|
||||
let selected = -1;
|
||||
let selectedHead: { id: string; bytes: Buffer } | undefined;
|
||||
for (let i = 0; i < heads.length; i++) {
|
||||
const candidate = heads[i];
|
||||
if (
|
||||
candidate &&
|
||||
(!selectedHead || Buffer.compare(candidate.bytes, selectedHead.bytes) < 0)
|
||||
) {
|
||||
selected = i;
|
||||
selectedHead = candidate;
|
||||
}
|
||||
}
|
||||
if (!selectedHead) break;
|
||||
const id = selectedHead.id;
|
||||
if (id === previous)
|
||||
throw new Error(
|
||||
'Embedding read: duplicate graph node identity; rebuild with analyze --force',
|
||||
);
|
||||
previous = id;
|
||||
const next = await readers[selected].next();
|
||||
heads[selected] = next.done ? undefined : head(next.value.text);
|
||||
}
|
||||
} finally {
|
||||
await Promise.all(readers.map((reader) => reader.return(undefined)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Preflight and publish the complete selection before any provider/write work. */
|
||||
export async function createEmbeddingWorkManifest(
|
||||
directory: string,
|
||||
sourceIdentity: string,
|
||||
source: EmbeddingManifestSource,
|
||||
): Promise<EmbeddingManifestReceipt> {
|
||||
if (!sourceIdentity || Buffer.byteLength(sourceIdentity) > 512)
|
||||
throw new Error('Invalid manifest source identity');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
const pendingPath = path.join(canonical, 'manifest.pending');
|
||||
const handle = await fs.open(pendingPath, 'wx', 0o600);
|
||||
const output = new BufferedLines(handle);
|
||||
const hash = createHash('sha256');
|
||||
const keyFiles: string[] = [];
|
||||
let totalNodes = 0;
|
||||
let totalUnits = 0;
|
||||
let maximumUnitBytes = 0;
|
||||
let bytes = 0;
|
||||
try {
|
||||
const scanLabel = async (label: string): Promise<void> => {
|
||||
const keyPath = path.join(canonical, `keys-${keyFiles.length}.jsonl`);
|
||||
const keysHandle = await fs.open(keyPath, 'wx', 0o600);
|
||||
keyFiles.push(keyPath);
|
||||
const keyOutput = new BufferedLines(keysHandle);
|
||||
let previous: Buffer | undefined;
|
||||
let nodes: EmbeddingWorkKey[] = [];
|
||||
let retainedBytes = 0;
|
||||
let recordBytes = 0;
|
||||
const flush = async (): Promise<void> => {
|
||||
if (!nodes.length) return;
|
||||
const unit: EmbeddingWorkUnit = { ordinal: totalUnits++, label, nodes, retainedBytes };
|
||||
const line = `${JSON.stringify(unit)}\n`;
|
||||
await output.write(line);
|
||||
hash.update(line);
|
||||
bytes += Buffer.byteLength(line);
|
||||
maximumUnitBytes = Math.max(maximumUnitBytes, retainedBytes);
|
||||
nodes = [];
|
||||
retainedBytes = 0;
|
||||
recordBytes = 0;
|
||||
};
|
||||
try {
|
||||
for await (const key of source.scan(label)) {
|
||||
validateKey(key);
|
||||
const id = Buffer.from(key.id);
|
||||
if (previous && Buffer.compare(previous, id) >= 0)
|
||||
throw new Error(
|
||||
`Embedding ${label} key scan is unordered or contains duplicate identities`,
|
||||
);
|
||||
previous = id;
|
||||
const serialized = `${JSON.stringify(key)}\n`;
|
||||
const size = Buffer.byteLength(serialized);
|
||||
if (size + 1024 > MAX_LINE_BYTES)
|
||||
throw new Error('Embedding identity exceeds the manifest record budget');
|
||||
if (
|
||||
nodes.length &&
|
||||
(nodes.length === UNIT_NODES ||
|
||||
retainedBytes + keyBytes(key) > UNIT_SOURCE_BYTES ||
|
||||
recordBytes + size + 1024 > MAX_LINE_BYTES)
|
||||
)
|
||||
await flush();
|
||||
await keyOutput.write(serialized);
|
||||
nodes.push({
|
||||
id: key.id,
|
||||
contentBytes: key.contentBytes,
|
||||
metadataBytes: key.metadataBytes,
|
||||
});
|
||||
retainedBytes += keyBytes(key);
|
||||
recordBytes += size;
|
||||
totalNodes++;
|
||||
}
|
||||
await flush();
|
||||
await keyOutput.flush();
|
||||
} finally {
|
||||
await keysHandle.close();
|
||||
}
|
||||
};
|
||||
for (const label of EMBEDDABLE_LABELS) await scanLabel(label);
|
||||
if (totalNodes === 0) await scanLabel('File');
|
||||
await verifyUniqueKeys(keyFiles);
|
||||
await output.flush();
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
const receipt: EmbeddingManifestReceipt = {
|
||||
version: 1,
|
||||
sourceIdentity,
|
||||
digest: hash.digest('hex'),
|
||||
bytes,
|
||||
totalNodes,
|
||||
totalUnits,
|
||||
maximumUnitBytes,
|
||||
};
|
||||
await fs.rename(pendingPath, path.join(canonical, 'manifest.jsonl'));
|
||||
await syncEmbeddingJobDirectory(canonical);
|
||||
await writeEmbeddingJobRecord(canonical, 'manifest.json', receipt);
|
||||
await Promise.all(keyFiles.map((file) => fs.unlink(file)));
|
||||
return receipt;
|
||||
}
|
||||
|
||||
/** Validate the durable selection without re-reading graph content or keys. */
|
||||
export async function loadEmbeddingWorkManifest(
|
||||
directory: string,
|
||||
sourceIdentity: string,
|
||||
): Promise<EmbeddingManifestReceipt> {
|
||||
const value = await readEmbeddingJobRecord(directory, 'manifest.json');
|
||||
if (!value || typeof value !== 'object') throw new Error('Malformed embedding manifest receipt');
|
||||
const receipt = value as EmbeddingManifestReceipt;
|
||||
if (
|
||||
receipt.version !== 1 ||
|
||||
receipt.sourceIdentity !== sourceIdentity ||
|
||||
!/^[0-9a-f]{64}$/.test(receipt.digest) ||
|
||||
!count(receipt.bytes) ||
|
||||
!count(receipt.totalNodes) ||
|
||||
!count(receipt.totalUnits) ||
|
||||
!count(receipt.maximumUnitBytes)
|
||||
) {
|
||||
throw new Error('Embedding manifest identity or version mismatch; artifacts retained');
|
||||
}
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
const hash = createHash('sha256');
|
||||
let bytes = 0;
|
||||
for await (const line of lines(path.join(canonical, 'manifest.jsonl'))) {
|
||||
hash.update(`${line.text}\n`);
|
||||
bytes = line.end;
|
||||
if (bytes > receipt.bytes) throw new Error('Embedding manifest size mismatch');
|
||||
}
|
||||
if (bytes !== receipt.bytes || hash.digest('hex') !== receipt.digest)
|
||||
throw new Error('Embedding manifest checksum mismatch; artifacts retained');
|
||||
return receipt;
|
||||
}
|
||||
|
||||
export async function* readEmbeddingWorkUnits(
|
||||
directory: string,
|
||||
receipt: EmbeddingManifestReceipt,
|
||||
cursor: EmbeddingManifestCursor = { unit: 0, offset: 0 },
|
||||
): AsyncGenerator<{ unit: EmbeddingWorkUnit; next: EmbeddingManifestCursor }> {
|
||||
if (
|
||||
!count(cursor.unit) ||
|
||||
cursor.unit > receipt.totalUnits ||
|
||||
!count(cursor.offset) ||
|
||||
cursor.offset > receipt.bytes
|
||||
)
|
||||
throw new Error('Invalid embedding manifest cursor');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
let ordinal = cursor.unit;
|
||||
for await (const line of lines(path.join(canonical, 'manifest.jsonl'), cursor.offset)) {
|
||||
const unit: EmbeddingWorkUnit = JSON.parse(line.text);
|
||||
if (
|
||||
unit.ordinal !== ordinal ||
|
||||
!LABELS.has(unit.label) ||
|
||||
!Array.isArray(unit.nodes) ||
|
||||
!unit.nodes.length ||
|
||||
unit.nodes.length > UNIT_NODES
|
||||
)
|
||||
throw new Error('Invalid embedding work unit');
|
||||
unit.nodes.forEach(validateKey);
|
||||
if (
|
||||
unit.retainedBytes !== unit.nodes.reduce((sum, key) => sum + keyBytes(key), 0) ||
|
||||
unit.retainedBytes > receipt.maximumUnitBytes
|
||||
)
|
||||
throw new Error('Invalid embedding unit byte accounting');
|
||||
ordinal++;
|
||||
yield { unit, next: { unit: ordinal, offset: line.end } };
|
||||
}
|
||||
if (ordinal !== receipt.totalUnits)
|
||||
throw new Error('Embedding manifest cursor does not cover the remaining units');
|
||||
}
|
||||
|
||||
export async function fetchEmbeddingWorkUnit(
|
||||
source: EmbeddingManifestSource,
|
||||
unit: EmbeddingWorkUnit,
|
||||
budget: EmbeddingByteBudget,
|
||||
): Promise<{ nodes: EmbeddableNode[]; release: () => void }> {
|
||||
const release = budget.reserve(unit.retainedBytes);
|
||||
try {
|
||||
const rows = await source.fetch(
|
||||
unit.label,
|
||||
unit.nodes.map((node) => node.id),
|
||||
);
|
||||
const byId = new Map(rows.map((row) => [row.id, row]));
|
||||
if (rows.length !== unit.nodes.length || byId.size !== rows.length)
|
||||
throw new Error('Embedding source returned missing or duplicate nodes');
|
||||
const nodes = unit.nodes.map((key) => {
|
||||
const node = byId.get(key.id);
|
||||
if (
|
||||
!node ||
|
||||
node.label !== unit.label ||
|
||||
typeof node.content !== 'string' ||
|
||||
Buffer.byteLength(node.content) !== key.contentBytes ||
|
||||
Buffer.byteLength(JSON.stringify({ ...node, content: undefined })) > key.metadataBytes
|
||||
)
|
||||
throw new Error('Embedding graph changed after manifest publication');
|
||||
return node;
|
||||
});
|
||||
return { nodes, release };
|
||||
} catch (error) {
|
||||
release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -46,6 +46,46 @@ interface HttpConfig {
|
|||
|
||||
export interface EmbeddingRequestOptions {
|
||||
signal?: AbortSignal;
|
||||
/** Encoded successful response bound, reserved by the calling job. */
|
||||
maxResponseBytes?: number;
|
||||
}
|
||||
|
||||
// A local resource limit aborts the request without retrying or penalising
|
||||
// endpoint health. resilientFetch treats AbortError DOMExceptions as neutral.
|
||||
class EmbeddingResponseBudgetError extends DOMException {
|
||||
constructor(message: string) {
|
||||
super(message, 'AbortError');
|
||||
}
|
||||
}
|
||||
|
||||
/** Bound encoded bytes before JSON decoding; Content-Length alone is untrusted. */
|
||||
async function readEmbeddingResponse(response: Response, maximum?: number): Promise<unknown> {
|
||||
if (maximum === undefined) return response.json();
|
||||
if (!Number.isSafeInteger(maximum) || maximum < 1)
|
||||
throw new Error('Invalid embedding response byte limit');
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) throw new Error('Embedding response has no body');
|
||||
let complete = false;
|
||||
try {
|
||||
const declared = response.headers.get('content-length');
|
||||
if (declared !== null && Number(declared) > maximum)
|
||||
throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget');
|
||||
const bytes = Buffer.allocUnsafe(maximum);
|
||||
let size = 0;
|
||||
for (;;) {
|
||||
const next = await reader.read();
|
||||
if (next.done) break;
|
||||
if (next.value.byteLength > maximum - size)
|
||||
throw new EmbeddingResponseBudgetError('Embedding response exceeds its resource budget');
|
||||
bytes.set(next.value, size);
|
||||
size += next.value.byteLength;
|
||||
}
|
||||
complete = true;
|
||||
return JSON.parse(new TextDecoder().decode(bytes.subarray(0, size)));
|
||||
} finally {
|
||||
if (!complete) await reader.cancel().catch(() => {});
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
let lastHttpRequestStartedAt: number | undefined;
|
||||
|
|
@ -427,6 +467,11 @@ const httpEmbedBatch = async (
|
|||
timeoutMs = DEFAULT_HTTP_TIMEOUT_MS,
|
||||
retryTimeouts = false,
|
||||
): Promise<EmbeddingItem[]> => {
|
||||
if (
|
||||
requestOptions.maxResponseBytes !== undefined &&
|
||||
(!Number.isSafeInteger(requestOptions.maxResponseBytes) || requestOptions.maxResponseBytes < 1)
|
||||
)
|
||||
throw new Error('Invalid embedding response byte limit');
|
||||
const requestBody: { input: string[]; model: string; dimensions?: number } = {
|
||||
input: batch,
|
||||
model,
|
||||
|
|
@ -490,8 +535,17 @@ const httpEmbedBatch = async (
|
|||
// and the same breaker accounting as any other endpoint fault (#2790).
|
||||
let payload: { data: EmbeddingItem[] };
|
||||
try {
|
||||
payload = (await attemptResp.json()) as { data: EmbeddingItem[] };
|
||||
payload = (await readEmbeddingResponse(
|
||||
attemptResp,
|
||||
requestOptions.maxResponseBytes,
|
||||
)) as {
|
||||
data: EmbeddingItem[];
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof EmbeddingResponseBudgetError)
|
||||
throw new EmbeddingResponseBudgetError(
|
||||
`Embedding endpoint exceeded its ${requestOptions.maxResponseBytes}-byte response budget (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
// Not every `.json()` rejection is a parse error: the per-attempt
|
||||
// signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is
|
||||
// wired to the body stream, so a stalled body rejects with the abort
|
||||
|
|
@ -540,6 +594,9 @@ const httpEmbedBatch = async (
|
|||
},
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof EmbeddingResponseBudgetError) {
|
||||
throw new HttpEmbeddingError(err.message, { cause: err });
|
||||
}
|
||||
if (
|
||||
requestOptions.signal?.aborted ||
|
||||
(err instanceof DOMException && err.name === 'AbortError')
|
||||
|
|
|
|||
|
|
@ -4,11 +4,15 @@ export interface ResolvedLineRange {
|
|||
}
|
||||
|
||||
export const buildLineIndex = (content: string): Int32Array => {
|
||||
const offsets: number[] = [0];
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
if (content.charCodeAt(i) === 10) offsets.push(i + 1);
|
||||
}
|
||||
return new Int32Array(offsets);
|
||||
// Allocate the final packed index directly; a temporary number[] costs much
|
||||
// more than the source for files with millions of short lines.
|
||||
let count = 1;
|
||||
for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) count++;
|
||||
const offsets = new Int32Array(count);
|
||||
let line = 1;
|
||||
for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1))
|
||||
offsets[line++] = i + 1;
|
||||
return offsets;
|
||||
};
|
||||
|
||||
const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readFile, realpath } from 'node:fs/promises';
|
||||
import { realpath } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { ContentRetention } from '../../storage/repo-meta.js';
|
||||
import { openEmbeddingJobFile } from '../../storage/embedding-job.js';
|
||||
import { isNotebookPath, notebookPythonSnippet } from '../ingestion/ipynb-extractor.js';
|
||||
import { buildLineIndex } from './line-index.js';
|
||||
import {
|
||||
|
|
@ -47,6 +48,7 @@ interface SourceFile {
|
|||
content: string;
|
||||
lines: Int32Array;
|
||||
bytes: number;
|
||||
sourceBytes: number;
|
||||
}
|
||||
|
||||
const CACHE_BYTES = 16 * 1024 * 1024;
|
||||
|
|
@ -58,9 +60,17 @@ export class EmbeddingSourceReader {
|
|||
private bytes = 0;
|
||||
private root: Promise<string> | undefined;
|
||||
|
||||
constructor(private readonly source: EmbeddingSourceContext) {}
|
||||
constructor(
|
||||
private readonly source: EmbeddingSourceContext,
|
||||
private readonly cacheBytes = CACHE_BYTES,
|
||||
) {}
|
||||
|
||||
async read(node: EmbeddableNode): Promise<string> {
|
||||
/** Admission runs before allocation, including on cache hits. The caller
|
||||
* holds its reservation until it finishes using/parsing the returned text. */
|
||||
async read(
|
||||
node: EmbeddableNode,
|
||||
admit?: (sourceBytes: number) => Promise<void>,
|
||||
): Promise<string> {
|
||||
if (this.source.contentRetention !== 'full') {
|
||||
throw new Error('Original embedding source requires full content retention');
|
||||
}
|
||||
|
|
@ -84,6 +94,7 @@ export class EmbeddingSourceReader {
|
|||
|
||||
let file = this.files.get(node.filePath);
|
||||
if (file) {
|
||||
await admit?.(file.sourceBytes);
|
||||
this.files.delete(node.filePath);
|
||||
this.files.set(node.filePath, file);
|
||||
} else {
|
||||
|
|
@ -107,15 +118,38 @@ export class EmbeddingSourceReader {
|
|||
) {
|
||||
throw new Error('source path leaves the repository');
|
||||
}
|
||||
const buffer = await readFile(resolved);
|
||||
const { handle, stat } = await openEmbeddingJobFile(resolved);
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
if (!Number.isSafeInteger(stat.size) || stat.size < 0)
|
||||
throw new Error('invalid source file size');
|
||||
await admit?.(stat.size);
|
||||
// Bound the read to the admitted size, plus one byte to detect growth.
|
||||
buffer = Buffer.alloc(stat.size + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset);
|
||||
if (!bytesRead) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
if (offset !== stat.size) throw new Error('source size changed during the read');
|
||||
buffer = buffer.subarray(0, offset);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
if (createHash('sha256').update(buffer).digest('hex') !== expectedHash) {
|
||||
throw new Error('checkout content differs from the indexed snapshot');
|
||||
}
|
||||
const content = buffer.toString('utf8');
|
||||
const lines = buildLineIndex(content);
|
||||
file = { content, lines, bytes: content.length * 2 + lines.byteLength };
|
||||
if (file.bytes <= CACHE_BYTES) {
|
||||
while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > CACHE_BYTES) {
|
||||
file = {
|
||||
content,
|
||||
lines,
|
||||
bytes: content.length * 2 + lines.byteLength,
|
||||
sourceBytes: buffer.length,
|
||||
};
|
||||
if (file.bytes <= this.cacheBytes) {
|
||||
while (this.files.size >= CACHE_FILES || this.bytes + file.bytes > this.cacheBytes) {
|
||||
const entry = this.files.entries().next().value;
|
||||
if (!entry) break;
|
||||
const [oldest, cached] = entry;
|
||||
|
|
|
|||
|
|
@ -5,11 +5,9 @@
|
|||
* to extract method and field names for embedding text generation.
|
||||
*/
|
||||
|
||||
import { getProviderForFileContent } from '../ingestion/languages/index.js';
|
||||
import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js';
|
||||
import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js';
|
||||
import type { LanguageProvider } from '../ingestion/language-provider.js';
|
||||
import { buildTypeEnv } from '../ingestion/type-env.js';
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import { ensureAndParse, findDeclarationNode } from './ast-utils.js';
|
||||
|
||||
|
|
@ -34,6 +32,7 @@ export const extractStructuralNames = async (
|
|||
parsedTree?: Awaited<ReturnType<typeof ensureAndParse>>,
|
||||
expectedName?: string,
|
||||
): Promise<StructuralNames> => {
|
||||
const { getProviderForFileContent } = await import('../ingestion/languages/index.js');
|
||||
const provider = getProviderForFileContent(filePath, content);
|
||||
if (!provider) return { methodNames: [], fieldNames: [] };
|
||||
|
||||
|
|
@ -47,7 +46,7 @@ export const extractStructuralNames = async (
|
|||
const language = provider.id;
|
||||
|
||||
const methodNames = extractMethodNames(classNode, provider, filePath, language);
|
||||
const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language);
|
||||
const fieldNames = await extractFieldNames(classNode, provider, tree, filePath, language);
|
||||
|
||||
return { methodNames, fieldNames };
|
||||
};
|
||||
|
|
@ -67,15 +66,16 @@ function extractMethodNames(
|
|||
return result.methods.map((m) => m.name);
|
||||
}
|
||||
|
||||
function extractFieldNames(
|
||||
async function extractFieldNames(
|
||||
classNode: any,
|
||||
provider: LanguageProvider,
|
||||
tree: any,
|
||||
filePath: string,
|
||||
language: SupportedLanguages,
|
||||
): string[] {
|
||||
): Promise<string[]> {
|
||||
if (!provider.fieldExtractor) return [];
|
||||
|
||||
const { buildTypeEnv } = await import('../ingestion/type-env.js');
|
||||
const typeEnv = buildTypeEnv(tree, language);
|
||||
|
||||
const context: FieldExtractorContext = {
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ const truncateDescription = (text: string, maxLength: number): string => {
|
|||
* Clean code content for embedding
|
||||
*/
|
||||
const cleanContent = (content: string): string => {
|
||||
// Keep this hot function small enough for V8 to inline ordinary snippets.
|
||||
if (content.length >= 16 * 1024) return cleanLargeContent(content);
|
||||
return content
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
|
|
@ -57,6 +59,46 @@ const cleanContent = (content: string): string => {
|
|||
.trim();
|
||||
};
|
||||
|
||||
const cleanLargeContent = (content: string): string => {
|
||||
const normalized = content.replace(/\r\n/g, '\n').replace(/\n{3,}/g, '\n\n');
|
||||
const output = new TextBlocks();
|
||||
for (const line of contentLines(normalized)) output.append(line.trimEnd());
|
||||
return output.finish().trim();
|
||||
};
|
||||
|
||||
/** Output-sized storage with a bounded number of temporary line strings. */
|
||||
class TextBlocks {
|
||||
private pending: string[] = [];
|
||||
private output = '';
|
||||
count = 0;
|
||||
append(line: string): void {
|
||||
if (this.count++) this.pending.push('\n');
|
||||
this.pending.push(line);
|
||||
if (this.pending.length >= 1024) this.flush();
|
||||
}
|
||||
private flush(): void {
|
||||
this.output += this.pending.join('');
|
||||
this.pending = [];
|
||||
}
|
||||
finish(): string {
|
||||
this.flush();
|
||||
return this.output;
|
||||
}
|
||||
}
|
||||
|
||||
function* contentLines(content: string): Generator<string> {
|
||||
let start = 0;
|
||||
for (;;) {
|
||||
const end = content.indexOf('\n', start);
|
||||
if (end < 0) {
|
||||
yield content.slice(start);
|
||||
return;
|
||||
}
|
||||
yield content.slice(start, end);
|
||||
start = end + 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact location signal for the embedding header: the last 1-2 path segments
|
||||
* (immediate parent dir + basename), never the full deep path.
|
||||
|
|
@ -254,25 +296,34 @@ const DECL_START_RE =
|
|||
* - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
|
||||
*/
|
||||
export const extractDeclarationOnly = (content: string): string => {
|
||||
const lines = content.split('\n');
|
||||
const declLines: string[] = [];
|
||||
const declLines = new TextBlocks();
|
||||
let depth = 0;
|
||||
let started = false;
|
||||
let classDepth = 0;
|
||||
let skipDepth = 0;
|
||||
|
||||
for (const [idx, line] of lines.entries()) {
|
||||
let nextLineStart = 0;
|
||||
for (const line of contentLines(content)) {
|
||||
nextLineStart += line.length + 1;
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (!started) {
|
||||
if (DECL_START_RE.test(trimmed)) {
|
||||
// Non-brace language check: current line or next 3 lines must have `{`
|
||||
const nextLines = lines.slice(idx + 1, idx + 4);
|
||||
if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) {
|
||||
return '';
|
||||
if (!trimmed.includes('{')) {
|
||||
let lookaheadEnd = nextLineStart;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const end = content.indexOf('\n', lookaheadEnd);
|
||||
if (end < 0) {
|
||||
lookaheadEnd = content.length;
|
||||
break;
|
||||
}
|
||||
lookaheadEnd = end + 1;
|
||||
}
|
||||
if (!content.slice(nextLineStart, lookaheadEnd).includes('{')) return '';
|
||||
}
|
||||
started = true;
|
||||
declLines.push(trimmed);
|
||||
declLines.append(trimmed);
|
||||
for (const ch of trimmed) {
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
|
|
@ -283,8 +334,12 @@ export const extractDeclarationOnly = (content: string): string => {
|
|||
}
|
||||
|
||||
// Always update depth (even when skipping)
|
||||
const opens = (trimmed.match(/{/g) || []).length;
|
||||
const closes = (trimmed.match(/}/g) || []).length;
|
||||
let opens = 0;
|
||||
let closes = 0;
|
||||
for (let i = 0; i < trimmed.length; i++) {
|
||||
if (trimmed.charCodeAt(i) === 123) opens++;
|
||||
else if (trimmed.charCodeAt(i) === 125) closes++;
|
||||
}
|
||||
const prevDepth = depth;
|
||||
depth += opens - closes;
|
||||
|
||||
|
|
@ -293,7 +348,7 @@ export const extractDeclarationOnly = (content: string): string => {
|
|||
skipDepth = 0;
|
||||
// Closing brace of class
|
||||
if (depth <= 0) {
|
||||
declLines.push(trimmed);
|
||||
declLines.append(trimmed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -306,7 +361,7 @@ export const extractDeclarationOnly = (content: string): string => {
|
|||
if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) {
|
||||
if (opens === closes && trimmed.endsWith(';')) {
|
||||
// Property with function/object initializer like `config = { timeout: 5000 };` — keep
|
||||
declLines.push(trimmed);
|
||||
declLines.append(trimmed);
|
||||
}
|
||||
// else: single-line or multi-line method — skip entirely
|
||||
if (opens !== closes) {
|
||||
|
|
@ -315,12 +370,12 @@ export const extractDeclarationOnly = (content: string): string => {
|
|||
continue;
|
||||
}
|
||||
|
||||
declLines.push(trimmed);
|
||||
declLines.append(trimmed);
|
||||
|
||||
if (depth <= 0 && declLines.length > 1) break;
|
||||
if (depth <= 0 && declLines.count > 1) break;
|
||||
}
|
||||
|
||||
return declLines.join('\n').trim();
|
||||
return declLines.finish().trim();
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
*
|
||||
* Type definitions for the embedding generation and semantic search system.
|
||||
*/
|
||||
import type { AnalysisPhaseProgress } from '../analysis-progress.js';
|
||||
|
||||
export const LABEL_FUNCTION = 'Function' as const;
|
||||
export const LABEL_METHOD = 'Method' as const;
|
||||
|
|
@ -206,7 +207,7 @@ export type EmbeddingPhase =
|
|||
/**
|
||||
* Progress information for the embedding pipeline
|
||||
*/
|
||||
export interface EmbeddingProgress {
|
||||
export interface EmbeddingProgress extends AnalysisPhaseProgress {
|
||||
phase: EmbeddingPhase;
|
||||
percent: number;
|
||||
modelDownloadPercent?: number;
|
||||
|
|
|
|||
138
gitnexus/src/core/embeddings/vector-generation-search.ts
Normal file
138
gitnexus/src/core/embeddings/vector-generation-search.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
import type { RepoMeta } from '../../storage/repo-meta.js';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { EmbeddingIdentity } from './embedding-identity.js';
|
||||
import {
|
||||
assertOwnedVectorGeneration,
|
||||
selectVectorGeneration,
|
||||
} from '../../storage/vector-generation.js';
|
||||
import {
|
||||
initLbug,
|
||||
pinRepo,
|
||||
executeParameterized,
|
||||
ensureVectorExtension,
|
||||
} from '../lbug/pool-adapter.js';
|
||||
import { VECTOR_STORE_SCHEMA } from '../lbug/vector-store.js';
|
||||
import { getExactScanLimit } from '../platform/capabilities.js';
|
||||
import { collectBestChunks } from './types.js';
|
||||
import { rankExactEmbeddingRows, type ExactSearchChunk } from './exact-search.js';
|
||||
import { assertQueryIdentity, rethrowSymbolIdentityError } from '../lbug/query-result-integrity.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
export interface VectorSearchSnapshot {
|
||||
metadataDirectory: string;
|
||||
metadata: RepoMeta | null;
|
||||
graphPath: string;
|
||||
}
|
||||
|
||||
const configuredIdentity = async (): Promise<EmbeddingIdentity> => {
|
||||
const { resolveEmbeddingIdentity } = await import('./embedding-identity.js');
|
||||
return resolveEmbeddingIdentity();
|
||||
};
|
||||
|
||||
/** null means a legacy monolithic reader is required. An empty array is a
|
||||
* deliberate unavailable lane and must not fall through to legacy vectors. */
|
||||
export async function searchVectorGeneration(
|
||||
snapshot: VectorSearchSnapshot,
|
||||
embed: () => Promise<readonly number[]>,
|
||||
limit: number,
|
||||
maxDistance: number,
|
||||
resolveIdentity: () => Promise<EmbeddingIdentity> = configuredIdentity,
|
||||
): Promise<ExactSearchChunk[] | null> {
|
||||
const exactLimit = getExactScanLimit();
|
||||
const selected = selectVectorGeneration(
|
||||
snapshot.metadataDirectory,
|
||||
snapshot.metadata,
|
||||
snapshot.graphPath,
|
||||
exactLimit,
|
||||
);
|
||||
if (selected.kind === 'legacy') return null;
|
||||
if (selected.kind !== 'ready') return [];
|
||||
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000 || !Number.isFinite(maxDistance))
|
||||
throw new Error('Invalid vector search bounds');
|
||||
const dbPath = selected.path;
|
||||
const artifactIdentity = selected.generation.artifactIdentity ?? selected.generation.identity;
|
||||
const release = pinRepo(dbPath);
|
||||
try {
|
||||
await assertOwnedVectorGeneration(snapshot.metadataDirectory, artifactIdentity);
|
||||
await initLbug(dbPath, dbPath, {
|
||||
kind: 'vector',
|
||||
bufferPoolBytes: 64 * 1024 * 1024,
|
||||
threads: 2,
|
||||
});
|
||||
const execute = (query: string, parameters: Record<string, unknown> = {}) =>
|
||||
executeParameterized(dbPath, query, parameters);
|
||||
const artifact = await execute(
|
||||
"MATCH (a:EmbeddingArtifact {id:'artifact'}) RETURN a.identity AS identity, a.schema AS schema, a.dimensions AS dimensions, a.complete AS complete, a.indexKind AS kind",
|
||||
);
|
||||
const proof = artifact[0];
|
||||
if (
|
||||
artifact.length !== 1 ||
|
||||
proof.identity !== artifactIdentity ||
|
||||
proof.schema !== VECTOR_STORE_SCHEMA ||
|
||||
Number(proof.dimensions) !== selected.generation.dimensions ||
|
||||
proof.complete !== true ||
|
||||
proof.kind !== selected.generation.indexKind
|
||||
)
|
||||
throw new Error('Published vector identity is unverified');
|
||||
const annReady =
|
||||
selected.generation.indexKind === 'hnsw' &&
|
||||
(await ensureVectorExtension(dbPath).catch((error: unknown) => {
|
||||
logger.warn(
|
||||
{ err: error },
|
||||
'Published VECTOR extension could not load; checking the bounded exact-search fallback',
|
||||
);
|
||||
return false;
|
||||
}));
|
||||
if (!annReady && selected.generation.chunks > exactLimit) return [];
|
||||
const identity = await resolveIdentity();
|
||||
const vectorIdentity = createHash('sha256')
|
||||
.update(JSON.stringify([identity.provider, identity.model, identity.dimensions]))
|
||||
.digest('hex');
|
||||
if (vectorIdentity !== selected.generation.vectorIdentity) return [];
|
||||
// Neither a metadata pointer nor a timestamp alone authorizes a model call.
|
||||
const query = await embed();
|
||||
if (
|
||||
query.length !== selected.generation.dimensions ||
|
||||
query.some((value) => !Number.isFinite(value))
|
||||
)
|
||||
throw new Error('Query embedding does not match the published vector dimensions');
|
||||
if (annReady) {
|
||||
try {
|
||||
const best = await collectBestChunks(limit, async (fetchLimit) => {
|
||||
const rows = await execute(
|
||||
`CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', $vector, $limit)
|
||||
YIELD node AS emb, distance WITH emb, distance WHERE distance < $distance
|
||||
RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, emb.startLine AS startLine, emb.endLine AS endLine, distance ORDER BY distance`,
|
||||
{ vector: [...query], limit: fetchLimit, distance: maxDistance },
|
||||
);
|
||||
return rows.map((row) => {
|
||||
assertQueryIdentity(row, 'nodeId', 0);
|
||||
return row as ExactSearchChunk;
|
||||
});
|
||||
});
|
||||
if (best.size > 0) return [...best].map(([nodeId, chunk]) => ({ nodeId, ...chunk }));
|
||||
} catch (error) {
|
||||
rethrowSymbolIdentityError(error);
|
||||
if (selected.generation.chunks > exactLimit) throw error;
|
||||
}
|
||||
}
|
||||
if (selected.generation.chunks > exactLimit) return [];
|
||||
// determinism: probe — fetch the entire verified set; reject partial or oversized coverage before ranking.
|
||||
const rows =
|
||||
await execute(`MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex,
|
||||
e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding LIMIT ${exactLimit + 1}`);
|
||||
if (rows.length !== selected.generation.chunks || rows.length > exactLimit)
|
||||
throw new Error('Published vector coverage does not match its receipt');
|
||||
for (const row of rows) assertQueryIdentity(row, 'nodeId', 0);
|
||||
const seen = new Set<string>();
|
||||
return rankExactEmbeddingRows(rows, query, rows.length, maxDistance)
|
||||
.filter((row) => {
|
||||
if (seen.has(row.nodeId)) return false;
|
||||
seen.add(row.nodeId);
|
||||
return true;
|
||||
})
|
||||
.slice(0, limit);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
108
gitnexus/src/core/embeddings/vector-publication.ts
Normal file
108
gitnexus/src/core/embeddings/vector-publication.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import path from 'node:path';
|
||||
import type { RepoMeta } from '../../storage/repo-meta.js';
|
||||
import {
|
||||
assertOwnedVectorGeneration,
|
||||
selectVectorGeneration,
|
||||
type PublishedVectorGeneration,
|
||||
} from '../../storage/vector-generation.js';
|
||||
import { VectorStore } from '../lbug/vector-store.js';
|
||||
import { syncPublishedMetadata } from '../analysis-publication.js';
|
||||
import { getExactScanLimit } from '../platform/capabilities.js';
|
||||
import { reclaimAnalysisArtifacts } from '../../storage/analysis-artifact-cleanup.js';
|
||||
import { readEmbeddingArtifactOwners } from '../../storage/embedding-artifact.js';
|
||||
|
||||
export interface VectorPublicationActions {
|
||||
loadMetadata: () => Promise<RepoMeta | null>;
|
||||
saveMetadata: (metadata: RepoMeta) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Caller holds the checkout's index lock and has closed the vector writer.
|
||||
* A failed verify/close/save leaves the job in indexing, with no paid replay.
|
||||
* null means ANN remains pending beyond the exact-scan policy limit.
|
||||
*/
|
||||
export async function publishVectorGeneration(
|
||||
metadataDirectory: string,
|
||||
generation: PublishedVectorGeneration,
|
||||
resources: { bufferPoolBytes: number; threads: number; batchBytes: number },
|
||||
actions: VectorPublicationActions,
|
||||
migratedLegacySource?: string,
|
||||
): Promise<RepoMeta | null> {
|
||||
const exactScanLimit = getExactScanLimit();
|
||||
if (generation.indexKind === 'exact' && generation.chunks > exactScanLimit) return null;
|
||||
const previous = await actions.loadMetadata();
|
||||
if (
|
||||
!previous?.graphGeneration ||
|
||||
previous.incrementalInProgress ||
|
||||
previous.graphGeneration.id !== generation.graphGeneration ||
|
||||
previous.embeddingJob?.identity !== generation.identity ||
|
||||
previous.embeddingJob?.graphGeneration !== generation.graphGeneration ||
|
||||
(previous.embeddingJob.artifactIdentity ?? previous.embeddingJob.identity) !==
|
||||
(generation.artifactIdentity ?? generation.identity)
|
||||
)
|
||||
throw new Error('Vector publication does not match the active graph job');
|
||||
const metadata: RepoMeta = {
|
||||
...previous,
|
||||
vectorGeneration: generation,
|
||||
embeddingJob: undefined,
|
||||
embeddingCheckpoint: undefined,
|
||||
embeddingDims: generation.dimensions,
|
||||
stats: { ...previous.stats, embeddings: generation.chunks },
|
||||
capabilities: {
|
||||
graph: previous.capabilities?.graph ?? { provider: 'ladybugdb', status: 'available' },
|
||||
fts: previous.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' },
|
||||
vectorSearch: {
|
||||
provider: generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan',
|
||||
status: generation.chunks
|
||||
? generation.indexKind === 'hnsw'
|
||||
? 'vector-index'
|
||||
: 'exact-scan'
|
||||
: 'unavailable',
|
||||
exactScanLimit,
|
||||
...(generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}),
|
||||
},
|
||||
},
|
||||
};
|
||||
const selection = selectVectorGeneration(
|
||||
metadataDirectory,
|
||||
metadata,
|
||||
path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'),
|
||||
exactScanLimit,
|
||||
);
|
||||
if (
|
||||
selection.kind !== 'ready' &&
|
||||
!(selection.kind === 'unavailable' && selection.reason === 'embeddings-empty')
|
||||
)
|
||||
throw new Error('Invalid vector publication receipt');
|
||||
const artifactIdentity = generation.artifactIdentity ?? generation.identity;
|
||||
const file = await assertOwnedVectorGeneration(metadataDirectory, artifactIdentity);
|
||||
const store = await VectorStore.open({
|
||||
directory: path.dirname(file),
|
||||
identity: artifactIdentity,
|
||||
dimensions: generation.dimensions,
|
||||
mode: 'read',
|
||||
...resources,
|
||||
});
|
||||
try {
|
||||
if (
|
||||
(await store.seal(generation.nodes - (generation.excludedNodes ?? 0), generation.chunks)) !==
|
||||
generation.indexKind
|
||||
)
|
||||
throw new Error('Published vector index does not match the native artifact');
|
||||
} finally {
|
||||
await store.close();
|
||||
}
|
||||
if (previous.legacyVectorSource && previous.legacyVectorSource === migratedLegacySource) {
|
||||
const owners = await readEmbeddingArtifactOwners(metadataDirectory);
|
||||
if (
|
||||
owners.some(
|
||||
(owner) => owner.artifactIdentity === artifactIdentity && owner.phase === 'sealed',
|
||||
) &&
|
||||
!owners.some((owner) => owner.fallbackLegacy === previous.legacyVectorSource)
|
||||
)
|
||||
delete metadata.legacyVectorSource;
|
||||
}
|
||||
await actions.saveMetadata(metadata);
|
||||
await syncPublishedMetadata(metadataDirectory, metadata);
|
||||
await reclaimAnalysisArtifacts(metadataDirectory, metadata);
|
||||
return metadata;
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ export const INDEX_INCOMPLETE_REASONS = [
|
|||
'incremental-in-progress',
|
||||
'embedding-checkpoint-pending',
|
||||
'embedding-count-unverified',
|
||||
'embedding-job-pending',
|
||||
'graph-write-collapsed',
|
||||
'scope-extraction-unverified',
|
||||
'scope-extraction-failed',
|
||||
|
|
@ -137,6 +138,7 @@ export function getIndexIncompleteReasons(
|
|||
RepoMeta,
|
||||
| 'incrementalInProgress'
|
||||
| 'embeddingCheckpoint'
|
||||
| 'embeddingJob'
|
||||
| 'graphWriteCollapsed'
|
||||
| 'scopeExtractionFailures'
|
||||
| 'scopeExtractionReceipt'
|
||||
|
|
@ -172,5 +174,18 @@ export function getIndexIncompleteReasons(
|
|||
: 'embedding-checkpoint-pending',
|
||||
);
|
||||
}
|
||||
if (meta?.embeddingJob) reasons.push('embedding-job-pending');
|
||||
return reasons;
|
||||
}
|
||||
|
||||
/** Embedding debt does not invalidate a published graph or require parsing. */
|
||||
export function getGraphIncompleteReasons(
|
||||
meta: Parameters<typeof getIndexIncompleteReasons>[0],
|
||||
): IndexIncompleteReason[] {
|
||||
return getIndexIncompleteReasons(meta).filter(
|
||||
(reason) =>
|
||||
reason !== 'embedding-checkpoint-pending' &&
|
||||
reason !== 'embedding-count-unverified' &&
|
||||
reason !== 'embedding-job-pending',
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -616,10 +616,23 @@ export async function runChunkedParseAndResolve(
|
|||
// shrink an incremental re-analyze: `totalParseable` counts every parseable
|
||||
// file in the scan, not the changed ones, so a warm run of a large repo still
|
||||
// spawns the full requested pool.
|
||||
const effectivePoolSize =
|
||||
const workerResources = options?.workerResources;
|
||||
if (
|
||||
totalParseable > 0 &&
|
||||
workerResources &&
|
||||
(!Number.isSafeInteger(workerResources.workers) ||
|
||||
workerResources.workers < 1 ||
|
||||
!Number.isSafeInteger(workerResources.workerHeapMb) ||
|
||||
workerResources.workerHeapMb < 512 ||
|
||||
(explicitPoolSize !== undefined && explicitPoolSize > workerResources.workers))
|
||||
)
|
||||
throw new Error('Parser worker configuration exceeds its admitted resource budget');
|
||||
const effectivePoolSize = Math.min(
|
||||
explicitPoolSize && explicitPoolSize > 0
|
||||
? Math.min(explicitPoolSize, Math.max(1, totalParseable))
|
||||
: Math.min(resolveAutoPoolSize(), workProportionalCap);
|
||||
: Math.min(resolveAutoPoolSize(), workProportionalCap),
|
||||
workerResources?.workers ?? Infinity,
|
||||
);
|
||||
// Deliberate over-subscription is the operator's call, so this warns rather
|
||||
// than caps — silently capping is what the override exists to stop. But an
|
||||
// exported `GITNEXUS_WORKER_POOL_SIZE` applies to EVERY analyze in a
|
||||
|
|
@ -725,6 +738,9 @@ export async function runChunkedParseAndResolve(
|
|||
// after it is initialized; `undefined` drives the worker no-store
|
||||
// fallback (return ParsedFiles in the result).
|
||||
workerPool = createWorkerPool(workerUrl, effectivePoolSize, {
|
||||
...(workerResources
|
||||
? { heapLimitMb: workerResources.workerHeapMb, requireDrain: true }
|
||||
: {}),
|
||||
parsedFileStoreStoragePath: parsedFileStorePath,
|
||||
// Durable, content-addressed shard dir for warm-cache reuse (#2038).
|
||||
// Initialized below before the chunk loop (same deferred-init pattern
|
||||
|
|
@ -817,6 +833,8 @@ export async function runChunkedParseAndResolve(
|
|||
let chunkCacheMisses = 0;
|
||||
let parseCacheHitFileCount = 0;
|
||||
let reparsedFileCount = 0;
|
||||
let parseFailure: unknown;
|
||||
let parsingFailed = false;
|
||||
|
||||
try {
|
||||
// U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch
|
||||
|
|
@ -1227,6 +1245,7 @@ export async function runChunkedParseAndResolve(
|
|||
}
|
||||
let missIdx = 0;
|
||||
for (const entry of round.entries) {
|
||||
options?.signal?.throwIfAborted();
|
||||
if (entry.kind === 'hit') {
|
||||
const chunkWorkerData = mergeChunkResults(
|
||||
graph,
|
||||
|
|
@ -1257,6 +1276,7 @@ export async function runChunkedParseAndResolve(
|
|||
* park it resolved for the next close to merge.
|
||||
*/
|
||||
const closeRound = async (): Promise<void> => {
|
||||
options?.signal?.throwIfAborted();
|
||||
const started = await startRound(roundEntries);
|
||||
roundEntries = [];
|
||||
roundBudget.reset();
|
||||
|
|
@ -1291,6 +1311,7 @@ export async function runChunkedParseAndResolve(
|
|||
};
|
||||
|
||||
for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
|
||||
options?.signal?.throwIfAborted();
|
||||
if (heapProbeEveryN > 0 && chunkIdx > 0 && chunkIdx % heapProbeEveryN === 0) {
|
||||
logHeapProbe(
|
||||
`parse-chunk-${chunkIdx}`,
|
||||
|
|
@ -1525,8 +1546,23 @@ export async function runChunkedParseAndResolve(
|
|||
`processRoutesFromExtracted: ${ms.toFixed(0)}ms (${allExtractedRoutes.length} routes)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
parseFailure = error;
|
||||
parsingFailed = true;
|
||||
throw error;
|
||||
} finally {
|
||||
await workerPool?.terminate();
|
||||
try {
|
||||
await workerPool?.terminate();
|
||||
} catch (drainError) {
|
||||
if (parsingFailed) {
|
||||
throw new AggregateError(
|
||||
[parseFailure, drainError],
|
||||
`Parsing failed (${parseFailure instanceof Error ? parseFailure.message : String(parseFailure)}) and native workers have not drained`,
|
||||
{ cause: parseFailure },
|
||||
);
|
||||
}
|
||||
throw drainError;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch calls + ORM queries were already extracted inside each worker
|
||||
|
|
|
|||
|
|
@ -203,6 +203,7 @@ export async function runPipeline(
|
|||
const results = new Map<string, PhaseResult<unknown>>(seed);
|
||||
|
||||
for (const phase of sorted) {
|
||||
ctx.options?.signal?.throwIfAborted();
|
||||
const start = Date.now();
|
||||
|
||||
if (isDev) {
|
||||
|
|
|
|||
|
|
@ -56,6 +56,8 @@ import {
|
|||
} from './pipeline-phases/index.js';
|
||||
|
||||
export interface PipelineOptions {
|
||||
/** Cooperative cancellation checked between parser dispatch and graph phases. */
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Skip MRO, community detection, and process extraction for faster test runs.
|
||||
* The `pruneLocalSymbols` phase still runs — it is graph construction (it cleans
|
||||
|
|
@ -255,6 +257,8 @@ export interface PipelineOptions {
|
|||
* analyze invocations.
|
||||
*/
|
||||
workerPoolSize?: number;
|
||||
/** Admitted ceiling, separate from an explicit worker count so tiny repos retain auto-sizing. */
|
||||
workerResources?: Readonly<{ workers: number; workerHeapMb: number }>;
|
||||
/**
|
||||
* Process-detection budget (#3313). Explicit `maxProcesses` replaces the
|
||||
* dynamic `symbols / 10` formula; the other three replace compiled defaults.
|
||||
|
|
@ -388,6 +392,7 @@ export const runPipelineFromRepo = async (
|
|||
onProgress: (progress: PipelineProgress) => void,
|
||||
options?: PipelineOptions,
|
||||
): Promise<PipelineResult> => {
|
||||
options?.signal?.throwIfAborted();
|
||||
const graph = createKnowledgeGraph();
|
||||
const pipelineStart = Date.now();
|
||||
|
||||
|
|
@ -427,6 +432,7 @@ export const runPipelineFromRepo = async (
|
|||
let results;
|
||||
try {
|
||||
results = await runPipeline(phases, ctx);
|
||||
options?.signal?.throwIfAborted();
|
||||
graphEmitManifest = graphEmitSink?.finalize();
|
||||
} finally {
|
||||
// Release per-pair fds when the pipeline threw before finalize ran.
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ export function heapPressureRemedy(
|
|||
if (heapLimitBytes < autoCapBytes * 0.9) {
|
||||
if (source === 'budget') {
|
||||
return (
|
||||
`This machine has more memory available: raise --memory-budget, or omit it ` +
|
||||
`This machine has more memory available: raise --memory-budget (and --resource-budget if set), or omit the limits ` +
|
||||
`so gitnexus sizes its heap to the machine automatically.`
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -220,6 +220,10 @@ export interface WorkerPoolStats {
|
|||
}
|
||||
|
||||
export interface WorkerPoolOptions {
|
||||
/** Per-run old-generation allocation; overrides the process-wide environment default. */
|
||||
heapLimitMb?: number;
|
||||
/** Refuse to release the parser reservation while a retired native worker still runs. */
|
||||
requireDrain?: boolean;
|
||||
subBatchSize?: number;
|
||||
subBatchMaxBytes?: number;
|
||||
subBatchIdleTimeoutMs?: number;
|
||||
|
|
@ -1053,6 +1057,7 @@ export const createWorkerPool = (
|
|||
|
||||
const size = poolSize ?? resolveAutoPoolSize();
|
||||
const poolOptions = resolveWorkerPoolOptions(options, size);
|
||||
const requireDrain = options?.requireDrain === true;
|
||||
// Production factory spawns with `{ stderr: true }` so a worker's crash
|
||||
// output is redirected to a `worker.stderr` stream we can tee + capture
|
||||
// (see captureWorkerStderr) and attach to readiness-failure messages —
|
||||
|
|
@ -1071,7 +1076,12 @@ export const createWorkerPool = (
|
|||
parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg
|
||||
? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines }
|
||||
: undefined;
|
||||
const workerHeapCapMb = resolveWorkerHeapCapMb(size);
|
||||
const workerHeapCapMb = options?.heapLimitMb ?? resolveWorkerHeapCapMb(size);
|
||||
if (
|
||||
options?.heapLimitMb !== undefined &&
|
||||
(!Number.isSafeInteger(workerHeapCapMb) || workerHeapCapMb < 512)
|
||||
)
|
||||
throw new Error('Parser worker heap allocation must be a safe integer of at least 512 MiB');
|
||||
// The 512MB per-worker floor exists so a worker can parse anything real,
|
||||
// but on a very small container a large pool of floored workers can still
|
||||
// overcommit total memory (#2649 review). Behavior is unchanged — deaths
|
||||
|
|
@ -1135,6 +1145,7 @@ export const createWorkerPool = (
|
|||
safePoint: Promise<void>;
|
||||
};
|
||||
const retiredWorkers = new Set<RetiredWorkerRecord>();
|
||||
const pendingWorkerReleases = new Map<Promise<void>, number>();
|
||||
const respawnCount: number[] = new Array(size).fill(0);
|
||||
const activeSlots: Set<number> = new Set();
|
||||
// Layer 3 (quarantine): tracked via the dedicated `quarantine.ts`
|
||||
|
|
@ -1184,7 +1195,31 @@ export const createWorkerPool = (
|
|||
): Promise<void> => {
|
||||
const retired = Array.from(retiredWorkers);
|
||||
await Promise.all([
|
||||
...liveWorkers.map((worker) => worker?.terminate().catch(() => undefined)),
|
||||
...liveWorkers.map(async (worker, workerIndex) => {
|
||||
if (!worker) return;
|
||||
let completeRelease!: () => void;
|
||||
const termination = new Promise<void>((resolve) => {
|
||||
completeRelease = resolve;
|
||||
});
|
||||
const released = () => {
|
||||
worker.off('exit', released);
|
||||
pendingWorkerReleases.delete(termination);
|
||||
completeRelease();
|
||||
};
|
||||
if (requireDrain) {
|
||||
pendingWorkerReleases.set(termination, workerIndex);
|
||||
worker.once('exit', released);
|
||||
}
|
||||
try {
|
||||
await worker.terminate();
|
||||
released();
|
||||
} catch {
|
||||
// A rejected request does not prove physical exit. Retain ownership
|
||||
// after workers[] is cleared, and release it on the eventual exit.
|
||||
if (!requireDrain) released();
|
||||
else worker.unref?.();
|
||||
}
|
||||
}),
|
||||
...retired.map(async (record) => {
|
||||
// #2432: a retired worker that has not reached a JS-visible safe
|
||||
// point may be inside an N-API call — terminating it aborts the
|
||||
|
|
@ -1492,14 +1527,14 @@ export const createWorkerPool = (
|
|||
reason: string,
|
||||
): void => {
|
||||
let cleaned = false;
|
||||
let terminateStarted = false;
|
||||
let termination: Promise<void> | undefined;
|
||||
let resolveSafePoint!: () => void;
|
||||
const safePoint = new Promise<void>((resolve) => {
|
||||
resolveSafePoint = resolve;
|
||||
});
|
||||
|
||||
// A message/messageerror proves the worker is executing JS again; an
|
||||
// exit/error means the thread is gone. Either way `worker.terminate()`
|
||||
// exit/error means native execution has stopped. Either way `worker.terminate()`
|
||||
// can no longer land mid-N-API call (#2432), so shutdown's drain may
|
||||
// stop waiting.
|
||||
function markSafeToTerminate() {
|
||||
|
|
@ -1517,11 +1552,14 @@ export const createWorkerPool = (
|
|||
retiredWorkers.delete(record);
|
||||
}
|
||||
|
||||
async function terminateRetired() {
|
||||
if (terminateStarted) return;
|
||||
terminateStarted = true;
|
||||
cleanupRetired();
|
||||
await worker.terminate().catch(() => undefined);
|
||||
function terminateRetired(): Promise<void> {
|
||||
termination ??= worker.terminate().then(
|
||||
() => cleanupRetired(),
|
||||
() => {
|
||||
if (!requireDrain) cleanupRetired();
|
||||
},
|
||||
);
|
||||
return termination;
|
||||
}
|
||||
|
||||
function terminateWhenBackInJs() {
|
||||
|
|
@ -1539,7 +1577,9 @@ export const createWorkerPool = (
|
|||
|
||||
const onRetiredError = () => {
|
||||
markSafeToTerminate();
|
||||
cleanupRetired();
|
||||
// The error event precedes physical exit. Keep the reservation and
|
||||
// exit listener if termination subsequently rejects or stalls.
|
||||
if (!requireDrain) cleanupRetired();
|
||||
};
|
||||
const onRetiredExit = () => {
|
||||
markSafeToTerminate();
|
||||
|
|
@ -1581,10 +1621,24 @@ export const createWorkerPool = (
|
|||
}
|
||||
// Recovery must settle before dispatch returns, but a failed thread
|
||||
// may never acknowledge termination. Bound that wait as in shutdown.
|
||||
const termination = existing.terminate().then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
let completeRelease!: () => void;
|
||||
const termination = new Promise<void>((resolve) => {
|
||||
completeRelease = resolve;
|
||||
});
|
||||
const released = () => {
|
||||
existing.off('exit', released);
|
||||
pendingWorkerReleases.delete(termination);
|
||||
completeRelease();
|
||||
};
|
||||
if (requireDrain) {
|
||||
pendingWorkerReleases.set(termination, workerIndex);
|
||||
// A rejected terminate request is not proof of release. An eventual
|
||||
// physical exit is, even when the termination promise rejected.
|
||||
existing.once('exit', released);
|
||||
}
|
||||
void existing.terminate().then(released, () => {
|
||||
if (!requireDrain) released();
|
||||
});
|
||||
if (!(await settledWithin(termination, poolOptions.shutdownDrainMs))) {
|
||||
existing.unref?.();
|
||||
logger.warn(
|
||||
|
|
@ -1601,6 +1655,22 @@ export const createWorkerPool = (
|
|||
): Promise<boolean> => {
|
||||
await removeWorkerFromSlot(workerIndex, mode, reason);
|
||||
if (stopped) return false;
|
||||
if (requireDrain) {
|
||||
// A timeout retires a native-active worker without killing it. Its
|
||||
// slot remains reserved until both the safe point and native exit.
|
||||
for (const record of retiredWorkers) {
|
||||
if (record.workerIndex !== workerIndex) continue;
|
||||
if (
|
||||
!record.safeToTerminate &&
|
||||
!(await settledWithin(record.safePoint, poolOptions.shutdownDrainMs))
|
||||
)
|
||||
return false;
|
||||
if (!(await settledWithin(record.terminate(), poolOptions.shutdownDrainMs)))
|
||||
return false;
|
||||
if (retiredWorkers.has(record)) return false;
|
||||
}
|
||||
if ([...pendingWorkerReleases.values()].includes(workerIndex) || stopped) return false;
|
||||
}
|
||||
const replacement = spawnAndCapture(workerUrl);
|
||||
try {
|
||||
await waitForWorkerReady(replacement, poolOptions.workerReadyTimeoutMs);
|
||||
|
|
@ -2418,6 +2488,12 @@ export const createWorkerPool = (
|
|||
await terminateTrackedWorkers(workers);
|
||||
workers.length = 0;
|
||||
activeSlots.clear();
|
||||
if (requireDrain && (retiredWorkers.size > 0 || pendingWorkerReleases.size > 0))
|
||||
throw new Error(
|
||||
'Parser workers have not released their native resource reservation; refusing to start another analysis phase. ' +
|
||||
`Suspect files: ${quarantine.snapshot().slice(0, 5).join(', ') || 'see the last parser progress message'}. ` +
|
||||
'Workers will exit at their next safe point. Exclude a repeatedly stuck file with .gitnexusignore and retry; do not kill a native worker thread.',
|
||||
);
|
||||
};
|
||||
|
||||
const dispatch = async <TInput, TResult>(
|
||||
|
|
|
|||
198
gitnexus/src/core/lbug/embedding-source.ts
Normal file
198
gitnexus/src/core/lbug/embedding-source.ts
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
/** Key-only manifest scans and bounded PK fetches against one immutable graph. */
|
||||
import lbug from '@ladybugdb/core';
|
||||
import {
|
||||
closeLbugConnection,
|
||||
openLbugConnection,
|
||||
type LbugConnectionHandle,
|
||||
} from './lbug-config.js';
|
||||
import {
|
||||
EMBEDDABLE_LABELS,
|
||||
LABELS_WITH_EXPORTED,
|
||||
LABEL_METHOD,
|
||||
type EmbeddableNode,
|
||||
} from '../embeddings/types.js';
|
||||
import type {
|
||||
EmbeddingManifestSource,
|
||||
EmbeddingWorkKey,
|
||||
} from '../embeddings/embedding-work-manifest.js';
|
||||
|
||||
const LABELS = new Set<string>([...EMBEDDABLE_LABELS, 'File']);
|
||||
// ECMAScript trim's whitespace set, including Unicode separators and BOM.
|
||||
// RE2's default \\S is ASCII-only and would change the File fallback selection.
|
||||
const NON_WHITESPACE =
|
||||
'[^\\x09-\\x0d\\x20\\x{a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]';
|
||||
|
||||
function checkedLabel(label: string): string {
|
||||
if (!LABELS.has(label)) throw new Error('Unsupported embedding manifest label');
|
||||
return `\`${label}\``;
|
||||
}
|
||||
|
||||
function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult {
|
||||
if (!Array.isArray(result)) return result;
|
||||
if (result.length === 1) return result[0];
|
||||
for (const part of result) part.close();
|
||||
throw new Error('Unexpected multiple graph query results');
|
||||
}
|
||||
|
||||
function size(value: unknown): number {
|
||||
if (typeof value !== 'number' && typeof value !== 'bigint')
|
||||
throw new Error('Invalid graph byte count');
|
||||
const result = Number(value);
|
||||
if (!Number.isSafeInteger(result) || result < 0) throw new Error('Invalid graph byte count');
|
||||
return result;
|
||||
}
|
||||
|
||||
function string(value: unknown): string {
|
||||
if (typeof value !== 'string') throw new Error('Invalid graph text field');
|
||||
return value;
|
||||
}
|
||||
function optionalString(value: unknown): string | undefined {
|
||||
return value == null ? undefined : string(value);
|
||||
}
|
||||
|
||||
export class EmbeddingGraphSource implements EmbeddingManifestSource {
|
||||
private active = false;
|
||||
private closed = false;
|
||||
private closing?: Promise<void>;
|
||||
private readonly fetchStatements = new Map<string, lbug.PreparedStatement>();
|
||||
private constructor(
|
||||
private readonly handle: LbugConnectionHandle,
|
||||
private readonly graphPath: string,
|
||||
) {}
|
||||
|
||||
/** graphPath has already passed the publication layer's ownership validation. */
|
||||
static async open(
|
||||
graphPath: string,
|
||||
bufferPoolBytes: number,
|
||||
threads: number,
|
||||
): Promise<EmbeddingGraphSource> {
|
||||
return new EmbeddingGraphSource(
|
||||
await openLbugConnection(lbug, graphPath, { readOnly: true, bufferPoolBytes, threads }),
|
||||
graphPath,
|
||||
);
|
||||
}
|
||||
|
||||
private acquire(): void {
|
||||
if (this.closed || this.active)
|
||||
throw new Error('Embedding graph source is closed or already in use');
|
||||
this.active = true;
|
||||
}
|
||||
|
||||
async *scan(label: string): AsyncGenerator<EmbeddingWorkKey> {
|
||||
const table = checkedLabel(label);
|
||||
this.acquire();
|
||||
let result: lbug.QueryResult | undefined;
|
||||
try {
|
||||
const fields = ['id', 'name', 'filePath'];
|
||||
if (label !== 'File') fields.push('description');
|
||||
if (label === LABEL_METHOD) fields.push('returnType');
|
||||
// JSON escaping costs at most six bytes per source byte. The fixed
|
||||
// allowance covers property names, numeric spans, booleans and the label.
|
||||
const metadataBytes = fields
|
||||
.map((field) => `octet_length(encode(coalesce(n.${field}, '')))`)
|
||||
.join(' + ');
|
||||
const filter =
|
||||
label === 'File'
|
||||
? "WHERE n.filePath IS NOT NULL AND n.filePath <> '' AND n.content <> '[Binary file - content not stored]' AND regexp_matches(coalesce(n.content,''), $nonWhitespace)"
|
||||
: '';
|
||||
const sql = `MATCH (n:${table}) ${filter} RETURN n.id AS id, octet_length(encode(coalesce(n.content,''))) AS contentBytes, 256 + 6 * (${metadataBytes}) AS metadataBytes ORDER BY n.id`;
|
||||
if (label === 'File') {
|
||||
const statement = await this.handle.conn.prepare(sql);
|
||||
if (!statement.isSuccess()) throw new Error(statement.getErrorMessage());
|
||||
result = single(
|
||||
await this.handle.conn.execute(statement, { nonWhitespace: NON_WHITESPACE }),
|
||||
);
|
||||
} else result = single(await this.handle.conn.query(sql));
|
||||
while (result.hasNext()) {
|
||||
const row = result.getNextSync();
|
||||
yield {
|
||||
id: string(row.id),
|
||||
contentBytes: size(row.contentBytes),
|
||||
metadataBytes: size(row.metadataBytes),
|
||||
};
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
result?.close();
|
||||
} finally {
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fetch(label: string, ids: readonly string[]): Promise<EmbeddableNode[]> {
|
||||
const table = checkedLabel(label);
|
||||
if (!ids.length || ids.length > 64 || new Set(ids).size !== ids.length)
|
||||
throw new Error('Invalid graph fetch unit');
|
||||
this.acquire();
|
||||
let result: lbug.QueryResult | undefined;
|
||||
try {
|
||||
let statement = this.fetchStatements.get(label);
|
||||
if (!statement) {
|
||||
const fields = ['id', 'name', 'filePath', 'content'];
|
||||
if (label !== 'File') fields.push('startLine', 'endLine', 'description');
|
||||
if (LABELS_WITH_EXPORTED.has(label)) fields.push('isExported');
|
||||
if (label === LABEL_METHOD) fields.push('parameterCount', 'returnType');
|
||||
statement = await this.handle.conn.prepare(
|
||||
`UNWIND $ids AS id MATCH (n:${table} {id:id}) RETURN ${fields.map((field) => `n.${field} AS ${field}`).join(', ')}`,
|
||||
);
|
||||
if (!statement.isSuccess()) throw new Error(statement.getErrorMessage());
|
||||
this.fetchStatements.set(label, statement);
|
||||
}
|
||||
result = single(await this.handle.conn.execute(statement, { ids: [...ids] }));
|
||||
const nodes: EmbeddableNode[] = [];
|
||||
while (result.hasNext()) {
|
||||
if (nodes.length >= ids.length) throw new Error('Graph fetch exceeded its manifest unit');
|
||||
const row = result.getNextSync();
|
||||
const content = row.content ?? '';
|
||||
if (typeof content !== 'string') throw new Error('Graph content is not text');
|
||||
let fileLines = 1;
|
||||
if (label === 'File')
|
||||
for (let i = 0; i < content.length; i++) if (content.charCodeAt(i) === 10) fileLines++;
|
||||
if (row.isExported != null && typeof row.isExported !== 'boolean')
|
||||
throw new Error('Invalid graph export flag');
|
||||
nodes.push({
|
||||
id: string(row.id),
|
||||
name: string(row.name),
|
||||
label,
|
||||
filePath: string(row.filePath),
|
||||
content,
|
||||
startLine: label === 'File' ? 1 : row.startLine == null ? undefined : size(row.startLine),
|
||||
endLine:
|
||||
label === 'File' ? fileLines : row.endLine == null ? undefined : size(row.endLine),
|
||||
...(label === 'File' ? {} : { description: optionalString(row.description) }),
|
||||
...(LABELS_WITH_EXPORTED.has(label)
|
||||
? { isExported: row.isExported == null ? undefined : (row.isExported as boolean) }
|
||||
: {}),
|
||||
...(label === LABEL_METHOD
|
||||
? {
|
||||
parameterCount: row.parameterCount == null ? undefined : size(row.parameterCount),
|
||||
returnType: optionalString(row.returnType),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
return nodes;
|
||||
} finally {
|
||||
try {
|
||||
result?.close();
|
||||
} finally {
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closing) return this.closing;
|
||||
if (this.active) throw new Error('Cannot close an active embedding graph read');
|
||||
this.closed = true;
|
||||
this.fetchStatements.clear();
|
||||
this.closing = closeLbugConnection(this.handle, { strict: true, databasePath: this.graphPath });
|
||||
try {
|
||||
await this.closing;
|
||||
} catch (error) {
|
||||
this.closing = undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +59,7 @@ import { resolveFtsVersionPair } from './vendored-extension-path.js';
|
|||
import {
|
||||
classifyDeleteAllError,
|
||||
closeLbugConnection,
|
||||
drainLbugResourceReleases,
|
||||
HANDLE_RELEASE_PROBE_ATTEMPTS,
|
||||
HANDLE_RELEASE_PROBE_DELAY_MS,
|
||||
isDbBusyError,
|
||||
|
|
@ -624,6 +625,7 @@ const writableFtsCrashWalEvidence = async (
|
|||
const reopenWritableAfterMissingShadow = async (
|
||||
dbPath: string,
|
||||
err: unknown,
|
||||
options: LbugInitOptions = {},
|
||||
): Promise<LbugConnectionHandle> => {
|
||||
// Analyze writers may park a leftover in-place FTS abort WAL. Serve/embed
|
||||
// refuse first via assertReadOnlyFtsCrashSafe and must not pass this
|
||||
|
|
@ -644,7 +646,12 @@ const reopenWritableAfterMissingShadow = async (
|
|||
throw new Error(renameFailureMessage(dbPath, renameErr));
|
||||
}
|
||||
|
||||
return await openLbugConnection(lbug, dbPath);
|
||||
return options.bufferPoolBytes !== undefined || options.threads !== undefined
|
||||
? await openLbugConnection(lbug, dbPath, {
|
||||
bufferPoolBytes: options.bufferPoolBytes,
|
||||
threads: options.threads,
|
||||
})
|
||||
: await openLbugConnection(lbug, dbPath);
|
||||
};
|
||||
|
||||
const ensureReadOnlyConnectionUsable = async (
|
||||
|
|
@ -802,8 +809,18 @@ const runSchemaCreationQueries = async (dbPath: string): Promise<unknown | null>
|
|||
return null;
|
||||
};
|
||||
|
||||
export const initLbug = async (dbPath: string, options: { skipFts?: boolean } = {}) => {
|
||||
return runWithSessionLock(() => ensureLbugInitialized(dbPath, options));
|
||||
export interface LbugInitOptions {
|
||||
skipFts?: boolean;
|
||||
bufferPoolBytes?: number;
|
||||
threads?: number;
|
||||
}
|
||||
|
||||
export const initLbug = async (dbPath: string, options: LbugInitOptions = {}) => {
|
||||
return runWithSessionLock(async () => {
|
||||
if (options.bufferPoolBytes !== undefined || options.threads !== undefined)
|
||||
await drainLbugResourceReleases();
|
||||
return ensureLbugInitialized(dbPath, options);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -856,34 +873,51 @@ export const withLbugDb = async <T>(
|
|||
};
|
||||
|
||||
let currentDbSkipFts = false;
|
||||
let currentDbBufferPoolBytes: number | undefined;
|
||||
let currentDbThreads: number | undefined;
|
||||
let nativeCloseUnverified = false;
|
||||
|
||||
const ensureLbugInitialized = async (
|
||||
dbPath: string,
|
||||
options: { readOnly?: boolean; skipFts?: boolean } = {},
|
||||
options: LbugInitOptions & { readOnly?: boolean } = {},
|
||||
) => {
|
||||
const readOnly = options.readOnly === true;
|
||||
const skipFts = options.skipFts === true;
|
||||
if (
|
||||
options.bufferPoolBytes !== undefined &&
|
||||
(!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes < 8 * 1024 * 1024)
|
||||
)
|
||||
throw new Error('Explicit LadybugDB buffer pool must be at least 8 MiB');
|
||||
if (
|
||||
options.threads !== undefined &&
|
||||
(!Number.isSafeInteger(options.threads) || options.threads < 1)
|
||||
)
|
||||
throw new Error('Explicit LadybugDB thread allocation must be a positive integer');
|
||||
if (
|
||||
conn &&
|
||||
!nativeCloseUnverified &&
|
||||
currentDbPath === dbPath &&
|
||||
currentDbReadOnly === readOnly &&
|
||||
currentDbSkipFts === skipFts
|
||||
currentDbSkipFts === skipFts &&
|
||||
currentDbBufferPoolBytes === options.bufferPoolBytes &&
|
||||
currentDbThreads === options.threads
|
||||
) {
|
||||
return { db, conn };
|
||||
}
|
||||
await doInitLbug(dbPath, { readOnly, skipFts });
|
||||
await doInitLbug(dbPath, { ...options, readOnly, skipFts });
|
||||
return { db, conn };
|
||||
};
|
||||
|
||||
const doInitLbug = async (
|
||||
dbPath: string,
|
||||
options: { readOnly?: boolean; skipFts?: boolean } = {},
|
||||
options: LbugInitOptions & { readOnly?: boolean } = {},
|
||||
) => {
|
||||
const readOnly = options.readOnly === true;
|
||||
const skipFts = options.skipFts === true;
|
||||
// Different database requested — close the old one first
|
||||
if (conn || db) {
|
||||
await safeClose();
|
||||
const resourceBound = options.bufferPoolBytes !== undefined || options.threads !== undefined;
|
||||
if (conn || db || nativeCloseUnverified || (resourceBound && currentDbPath)) {
|
||||
await safeClose({ strict: resourceBound });
|
||||
currentDbPath = null;
|
||||
ftsLoaded = false;
|
||||
vectorExtensionLoaded = false;
|
||||
|
|
@ -1045,10 +1079,19 @@ const doInitLbug = async (
|
|||
});
|
||||
|
||||
try {
|
||||
const opened = await openLbugConnection(lbug, dbPath);
|
||||
const opened = resourceBound
|
||||
? await openLbugConnection(lbug, dbPath, {
|
||||
bufferPoolBytes: options.bufferPoolBytes,
|
||||
threads: options.threads,
|
||||
})
|
||||
: await openLbugConnection(lbug, dbPath);
|
||||
db = opened.db;
|
||||
conn = opened.conn;
|
||||
currentDbReadOnly = false;
|
||||
// Record the live allocation before schema work, so an initialization
|
||||
// failure must also prove release before another native open.
|
||||
currentDbBufferPoolBytes = options.bufferPoolBytes;
|
||||
currentDbThreads = options.threads;
|
||||
} catch (err) {
|
||||
// Incremental analyze can hit a storage-version mismatch on construct
|
||||
// or (more often) on the first schema query below. Fail immediately
|
||||
|
|
@ -1066,7 +1109,7 @@ const doInitLbug = async (
|
|||
if (missingShadowError) {
|
||||
await safeClose();
|
||||
resetOpenConnectionState();
|
||||
const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError);
|
||||
const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError, options);
|
||||
db = reopened.db;
|
||||
conn = reopened.conn;
|
||||
currentDbReadOnly = false;
|
||||
|
|
@ -1094,6 +1137,8 @@ const doInitLbug = async (
|
|||
}
|
||||
|
||||
currentDbSkipFts = skipFts;
|
||||
currentDbBufferPoolBytes = options.bufferPoolBytes;
|
||||
currentDbThreads = options.threads;
|
||||
currentDbPath = dbPath;
|
||||
return { db, conn };
|
||||
};
|
||||
|
|
@ -2399,7 +2444,9 @@ export const fetchExistingEmbeddingHashes = async (
|
|||
*/
|
||||
export const flushWAL = async (): Promise<void> => {
|
||||
const c = conn;
|
||||
if (!c) return;
|
||||
// CHECKPOINT on a read-only connection can leave LadybugDB's checkpoint
|
||||
// marker set even though the write is rejected, poisoning the next reader.
|
||||
if (!c || currentDbReadOnly) return;
|
||||
try {
|
||||
await withConnLock(async () => {
|
||||
const checkpointResult = await c.query('CHECKPOINT');
|
||||
|
|
@ -2428,7 +2475,7 @@ export const flushWAL = async (): Promise<void> => {
|
|||
*/
|
||||
export const tryFlushWAL = async (): Promise<boolean> => {
|
||||
const c = conn;
|
||||
if (!c) return false;
|
||||
if (!c || currentDbReadOnly) return false;
|
||||
// Runs on the periodic WAL-checkpoint driver. The lock makes this CHECKPOINT
|
||||
// wait for any in-flight COPY / writeback on the singleton connection instead
|
||||
// of executing concurrently with it (the `analyze --pdg` heap-corruption bug).
|
||||
|
|
@ -2449,7 +2496,13 @@ export const tryFlushWAL = async (): Promise<boolean> => {
|
|||
* @see flushWAL — CHECKPOINT-only (connection stays open)
|
||||
* @see closeLbug — safeClose + module state reset (full teardown)
|
||||
*/
|
||||
export const safeClose = async (): Promise<void> => {
|
||||
export const safeClose = async (options: { strict?: boolean } = {}): Promise<void> => {
|
||||
const strict =
|
||||
options.strict === true ||
|
||||
nativeCloseUnverified ||
|
||||
currentDbBufferPoolBytes !== undefined ||
|
||||
currentDbThreads !== undefined;
|
||||
if (strict) nativeCloseUnverified = true;
|
||||
await flushWAL();
|
||||
// Capture before close — currentDbPath stays set so the Windows post-close
|
||||
// probe below knows which file to wait on.
|
||||
|
|
@ -2458,7 +2511,8 @@ export const safeClose = async (): Promise<void> => {
|
|||
try {
|
||||
// eslint-disable-next-line no-restricted-syntax -- sole authorised close site
|
||||
await conn.close();
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (strict) throw error;
|
||||
/* best-effort */
|
||||
}
|
||||
conn = null;
|
||||
|
|
@ -2467,7 +2521,8 @@ export const safeClose = async (): Promise<void> => {
|
|||
try {
|
||||
// eslint-disable-next-line no-restricted-syntax -- sole authorised close site
|
||||
await db.close();
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (strict) throw error;
|
||||
/* best-effort */
|
||||
}
|
||||
db = null;
|
||||
|
|
@ -2480,6 +2535,7 @@ export const safeClose = async (): Promise<void> => {
|
|||
if (process.platform === 'win32' && closingDbPath) {
|
||||
const released = await waitForWindowsHandleRelease(closingDbPath);
|
||||
if (!released) {
|
||||
if (strict) throw new Error('LadybugDB native resource reservation has not been released');
|
||||
// Probe exhausted with a lock code still in flight. The next
|
||||
// openLbugConnection will absorb whatever residual lag remains, but
|
||||
// a chronic warning helps operators spot AV interference (Windows
|
||||
|
|
@ -2493,6 +2549,7 @@ export const safeClose = async (): Promise<void> => {
|
|||
if (closingDbPath) {
|
||||
await finalizeLbugSidecarsAfterClose(closingDbPath, { logger });
|
||||
}
|
||||
nativeCloseUnverified = false;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -2528,15 +2585,26 @@ export const closeLbugBeforeExit = async (): Promise<void> => {
|
|||
// double-free.
|
||||
};
|
||||
|
||||
export const closeLbug = async (): Promise<void> => {
|
||||
await safeClose();
|
||||
export const closeLbug = async (options: { strict?: boolean } = {}): Promise<void> => {
|
||||
await safeClose(options);
|
||||
currentDbPath = null;
|
||||
currentDbBufferPoolBytes = undefined;
|
||||
currentDbThreads = undefined;
|
||||
ftsLoaded = false;
|
||||
vectorExtensionLoaded = false;
|
||||
vectorIndexEnsured = false;
|
||||
ensuredFTSIndexes.clear();
|
||||
};
|
||||
|
||||
/** A new analysis phase may reuse the prior phase's reservation only after its
|
||||
* native handles are gone. Wait for session work; never close an active query. */
|
||||
export const drainAnalysisNativeResources = async (): Promise<void> => {
|
||||
await runWithSessionLock(async () => {
|
||||
if (conn || db || nativeCloseUnverified) await closeLbug({ strict: true });
|
||||
await drainLbugResourceReleases();
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Thrown by {@link wipeLbugDbFiles} when a data-bearing member of the
|
||||
* LadybugDB file family is still present after the bounded
|
||||
|
|
|
|||
|
|
@ -847,6 +847,9 @@ type LbugModule = typeof lbug;
|
|||
export interface LbugDatabaseOptions {
|
||||
readOnly?: boolean;
|
||||
throwOnWalReplayFailure?: boolean;
|
||||
/** Explicit per-handle reservation; omitted preserves the existing policy. */
|
||||
bufferPoolBytes?: number;
|
||||
threads?: number;
|
||||
}
|
||||
|
||||
export interface LbugConnectionHandle {
|
||||
|
|
@ -854,6 +857,61 @@ export interface LbugConnectionHandle {
|
|||
conn: lbug.Connection;
|
||||
}
|
||||
|
||||
interface NativeRelease {
|
||||
db: lbug.Database;
|
||||
conn?: lbug.Connection;
|
||||
databasePath?: string;
|
||||
connectionClosed: boolean;
|
||||
databaseClosed: boolean;
|
||||
pending?: Promise<void>;
|
||||
}
|
||||
|
||||
// Only closing/failed handles are held strongly. A rejected close is not proof
|
||||
// that its native pool was released, including failed connection construction.
|
||||
const nativeReleases = new Map<lbug.Database, NativeRelease>();
|
||||
const reservedHandlePaths = new WeakMap<LbugConnectionHandle, string>();
|
||||
|
||||
function releaseNativeResources(
|
||||
handle: Pick<LbugConnectionHandle, 'db'> & Partial<Pick<LbugConnectionHandle, 'conn'>>,
|
||||
databasePath?: string,
|
||||
): Promise<void> {
|
||||
let release = nativeReleases.get(handle.db);
|
||||
if (!release) {
|
||||
release = { ...handle, databasePath, connectionClosed: !handle.conn, databaseClosed: false };
|
||||
nativeReleases.set(handle.db, release);
|
||||
}
|
||||
if (release.pending) return release.pending;
|
||||
const state = release;
|
||||
state.pending = (async () => {
|
||||
if (!state.connectionClosed) {
|
||||
if (state.conn) await state.conn.close();
|
||||
state.connectionClosed = true;
|
||||
}
|
||||
if (!state.databaseClosed) {
|
||||
await state.db.close();
|
||||
state.databaseClosed = true;
|
||||
}
|
||||
if (
|
||||
process.platform === 'win32' &&
|
||||
state.databasePath &&
|
||||
!(await waitForWindowsHandleRelease(state.databasePath))
|
||||
)
|
||||
throw new Error('LadybugDB native resource reservation has not been released');
|
||||
nativeReleases.delete(state.db);
|
||||
})().finally(() => {
|
||||
state.pending = undefined;
|
||||
});
|
||||
return state.pending;
|
||||
}
|
||||
|
||||
/** Before admitting another analysis phase, retry only previously requested
|
||||
* closes. Active readers/writers are never closed or interrupted here. */
|
||||
export async function drainLbugResourceReleases(): Promise<void> {
|
||||
for (const release of nativeReleases.values()) {
|
||||
await releaseNativeResources(release, release.databasePath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the error message indicates that a LadybugDB write
|
||||
* transaction could not proceed due to lock contention — either a file
|
||||
|
|
@ -946,10 +1004,15 @@ export function createLbugDatabase(
|
|||
databasePath: string,
|
||||
options: LbugDatabaseOptions = {},
|
||||
): lbug.Database {
|
||||
if (
|
||||
options.bufferPoolBytes !== undefined &&
|
||||
(!Number.isSafeInteger(options.bufferPoolBytes) || options.bufferPoolBytes <= 0)
|
||||
)
|
||||
throw new Error('Explicit LadybugDB buffer pool must be a positive byte count');
|
||||
// .d.ts declares fewer args than the native constructor accepts.
|
||||
return new (lbugModule.Database as any)(
|
||||
databasePath,
|
||||
resolveBufferManagerSize(), // bufferManagerSize (#2557: default min(2 GiB, 80% RAM); GITNEXUS_LBUG_BUFFER_POOL_SIZE overrides; 0 restores the native 80%-of-RAM default)
|
||||
options.bufferPoolBytes ?? resolveBufferManagerSize(), // Existing callers retain the environment/default policy.
|
||||
false, // enableCompression (pinned for v0.16.0)
|
||||
options.readOnly ?? false,
|
||||
LBUG_MAX_DB_SIZE,
|
||||
|
|
@ -1145,18 +1208,50 @@ export async function openLbugConnection(
|
|||
databasePath: string,
|
||||
options: LbugDatabaseOptions = {},
|
||||
): Promise<LbugConnectionHandle> {
|
||||
if (
|
||||
options.threads !== undefined &&
|
||||
(!Number.isSafeInteger(options.threads) || options.threads < 1)
|
||||
)
|
||||
throw new Error('LadybugDB threads must be a positive integer');
|
||||
const reserved = options.bufferPoolBytes !== undefined || options.threads !== undefined;
|
||||
if (reserved) await drainLbugResourceReleases();
|
||||
const safePath = toNativeSafePath(databasePath);
|
||||
let db: lbug.Database | undefined;
|
||||
try {
|
||||
db = await openWithLockRetry(() => createLbugDatabase(lbugModule, safePath, options), safePath);
|
||||
return { db, conn: new lbugModule.Connection(db) };
|
||||
const conn =
|
||||
options.threads === undefined
|
||||
? new lbugModule.Connection(db)
|
||||
: new lbugModule.Connection(db, options.threads);
|
||||
const handle = { db, conn };
|
||||
if (reserved) reservedHandlePaths.set(handle, safePath);
|
||||
return handle;
|
||||
} catch (err) {
|
||||
if (db) await db.close().catch(() => {});
|
||||
if (db) {
|
||||
if (reserved) {
|
||||
try {
|
||||
await releaseNativeResources({ db }, safePath);
|
||||
} catch (releaseError) {
|
||||
throw new AggregateError(
|
||||
[err, releaseError],
|
||||
'LadybugDB open failed and resource release is unverified',
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
} else await db.close().catch(() => {});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeLbugConnection(handle: LbugConnectionHandle): Promise<void> {
|
||||
export async function closeLbugConnection(
|
||||
handle: LbugConnectionHandle,
|
||||
options: { strict?: boolean; databasePath?: string } = {},
|
||||
): Promise<void> {
|
||||
if (options.strict || reservedHandlePaths.has(handle) || nativeReleases.has(handle.db)) {
|
||||
await releaseNativeResources(handle, options.databasePath ?? reservedHandlePaths.get(handle));
|
||||
return;
|
||||
}
|
||||
await handle.conn.close().catch(() => {});
|
||||
await handle.db.close().catch(() => {});
|
||||
}
|
||||
|
|
|
|||
185
gitnexus/src/core/lbug/legacy-vector-export.ts
Normal file
185
gitnexus/src/core/lbug/legacy-vector-export.ts
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
/** Disk-backed migration of the legacy vector table, without copying its graph. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import { openLbugConnection, closeLbugConnection, toNativeSafePath } from './lbug-config.js';
|
||||
import { escapeCypherString } from './cypher-escape.js';
|
||||
import { assertLbugPublicationReady } from './publication.js';
|
||||
import { embeddingJobDirectory, syncEmbeddingJobDirectory } from '../../storage/embedding-job.js';
|
||||
import {
|
||||
recordLegacyEmbeddingSource,
|
||||
type LegacyEmbeddingSourceRecord,
|
||||
} from '../../storage/legacy-embedding-source.js';
|
||||
|
||||
export interface LegacyVectorExportOptions {
|
||||
metadataDirectory: string;
|
||||
/** The caller owns the slot lock and a stable source connection. Only catalog,
|
||||
* count and native COPY results cross this callback, never vector payloads. */
|
||||
query: (cypher: string) => Promise<unknown[]>;
|
||||
provenance: Pick<
|
||||
LegacyEmbeddingSourceRecord,
|
||||
'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity'
|
||||
>;
|
||||
excluded: Iterable<string> | AsyncIterable<string>;
|
||||
/** Additional native pool, admitted alongside the source's pool. */
|
||||
bufferPoolBytes: number;
|
||||
threads: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const fields = ['id', 'nodeId', 'chunkIndex', 'startLine', 'endLine', 'embedding', 'contentHash'];
|
||||
const countQuery = 'MATCH (e:CodeEmbedding) RETURN count(e) AS count';
|
||||
|
||||
/** Preserve the old physical width before writable schema initialization can
|
||||
* replace its table. The caller publishes the returned receipt under its lock. */
|
||||
export async function exportLegacyVectorsFromFile(
|
||||
sourcePath: string,
|
||||
options: Omit<LegacyVectorExportOptions, 'query'> & { sourcePoolBytes: number },
|
||||
): Promise<string | undefined> {
|
||||
options.signal?.throwIfAborted();
|
||||
const stat = await fs.lstat(sourcePath);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Legacy vector source must be an owned regular file');
|
||||
const handle = await openLbugConnection(lbug, sourcePath, {
|
||||
readOnly: true,
|
||||
bufferPoolBytes: options.sourcePoolBytes,
|
||||
threads: options.threads,
|
||||
});
|
||||
try {
|
||||
return await exportLegacyVectors({
|
||||
...options,
|
||||
query: async (sql) => {
|
||||
const result = await handle.conn.query(sql);
|
||||
const cursors = Array.isArray(result) ? result : [result];
|
||||
try {
|
||||
if (cursors.length !== 1) throw new Error('Unexpected legacy vector export result');
|
||||
return await cursors[0].getAll();
|
||||
} finally {
|
||||
for (const cursor of cursors) cursor.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await closeLbugConnection(handle, { strict: true, databasePath: sourcePath });
|
||||
}
|
||||
}
|
||||
|
||||
function count(rows: unknown[]): number {
|
||||
const value = (rows[0] as { count?: unknown } | undefined)?.count;
|
||||
if (rows.length !== 1 || (typeof value !== 'number' && typeof value !== 'bigint'))
|
||||
throw new Error('Legacy vector export count is unverified');
|
||||
const n = Number(value);
|
||||
if (!Number.isSafeInteger(n) || n < 0) throw new Error('Legacy vector export count is invalid');
|
||||
return n;
|
||||
}
|
||||
|
||||
/** The returned receipt pins an independent, immutable table-only database.
|
||||
* Group completeness/input compatibility is still checked by LegacyVectorSource;
|
||||
* copying a row does not certify its historical provider or text configuration.
|
||||
* The source and its metadata are never changed here. Publish the returned
|
||||
* reference under the slot lock before any source mutation or cleanup. */
|
||||
export async function exportLegacyVectors(
|
||||
options: LegacyVectorExportOptions,
|
||||
): Promise<string | undefined> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (
|
||||
!Number.isSafeInteger(options.bufferPoolBytes) ||
|
||||
options.bufferPoolBytes < 8 * 1024 * 1024 ||
|
||||
!Number.isSafeInteger(options.threads) ||
|
||||
options.threads < 1
|
||||
)
|
||||
throw new Error('Legacy vector export requires an admitted native pool');
|
||||
const root = await embeddingJobDirectory(options.metadataDirectory);
|
||||
const expected = count(await options.query(countQuery));
|
||||
if (expected === 0) return undefined;
|
||||
const catalog = (await options.query("CALL table_info('CodeEmbedding') RETURN *")) as {
|
||||
name?: unknown;
|
||||
type?: unknown;
|
||||
'primary key'?: unknown;
|
||||
}[];
|
||||
const vectorType = catalog.find((column) => column.name === 'embedding')?.type;
|
||||
const match = typeof vectorType === 'string' && /^FLOAT\[([1-9]\d*)\]$/.exec(vectorType);
|
||||
const dimensions = match ? Number(match[1]) : NaN;
|
||||
const types = ['STRING', 'STRING', 'INT32', 'INT64', 'INT64', vectorType, 'STRING'];
|
||||
if (
|
||||
!Number.isSafeInteger(dimensions) ||
|
||||
dimensions < 1 ||
|
||||
fields.some((name, i) => !catalog.some((c) => c.name === name && c.type === types[i])) ||
|
||||
!catalog.some((c) => c.name === 'id' && c['primary key'] === true) ||
|
||||
(options.provenance.recordedIdentity?.dimensions !== undefined &&
|
||||
options.provenance.recordedIdentity.dimensions !== dimensions)
|
||||
)
|
||||
throw new Error('Legacy vector schema differs from its recorded provenance; source retained');
|
||||
|
||||
const id = randomUUID();
|
||||
const staging = path.join(root, `lbug.staging.${id}`);
|
||||
const csv = staging + '.csv';
|
||||
const sourceFile = `lbug.generation.${id.replaceAll('-', '')}`;
|
||||
const destination = path.join(root, sourceFile);
|
||||
const csvLiteral = `'${escapeCypherString(toNativeSafePath(csv).replace(/\\/g, '/'))}'`;
|
||||
let stagingReleased = true;
|
||||
try {
|
||||
options.signal?.throwIfAborted();
|
||||
await options.query(
|
||||
`COPY (MATCH (e:CodeEmbedding) RETURN ${fields.map((f) => `e.${f} AS ${f}`).join(', ')}) TO ${csvLiteral} (HEADER=true)`,
|
||||
);
|
||||
options.signal?.throwIfAborted();
|
||||
stagingReleased = false;
|
||||
const handle = await openLbugConnection(lbug, staging, {
|
||||
bufferPoolBytes: options.bufferPoolBytes,
|
||||
threads: options.threads,
|
||||
});
|
||||
const query = async (sql: string): Promise<unknown[]> => {
|
||||
const result = await handle.conn.query(sql);
|
||||
const results = Array.isArray(result) ? result : [result];
|
||||
try {
|
||||
if (results.length !== 1) throw new Error('Unexpected legacy vector export result');
|
||||
return await results[0].getAll();
|
||||
} finally {
|
||||
for (const cursor of results) cursor.close();
|
||||
}
|
||||
};
|
||||
try {
|
||||
await query(
|
||||
`CREATE NODE TABLE CodeEmbedding (${fields.map((name, i) => `${name} ${types[i]}`).join(', ')}, PRIMARY KEY(id))`,
|
||||
);
|
||||
await query(`COPY CodeEmbedding FROM ${csvLiteral} (HEADER=true, auto_detect=false)`);
|
||||
if (
|
||||
count(await query(countQuery)) !== expected ||
|
||||
count(await options.query(countQuery)) !== expected
|
||||
)
|
||||
throw new Error('Legacy vector export lost rows or its source changed; source retained');
|
||||
options.signal?.throwIfAborted();
|
||||
await query('CHECKPOINT');
|
||||
} finally {
|
||||
// A failed physical release remains charged by the native reservation
|
||||
// manager. Never rename or certify that still-open export.
|
||||
await closeLbugConnection(handle, { strict: true, databasePath: staging });
|
||||
stagingReleased = true;
|
||||
}
|
||||
options.signal?.throwIfAborted();
|
||||
await assertLbugPublicationReady(staging);
|
||||
await fs.rename(staging, destination);
|
||||
const file = await fs.open(destination, 'r+');
|
||||
try {
|
||||
await file.sync();
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
return await recordLegacyEmbeddingSource(
|
||||
root,
|
||||
{ sourceFile, ...options.provenance },
|
||||
options.excluded,
|
||||
options.signal,
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(csv, { force: true });
|
||||
// The original source remains untouched. A failed close must retain its
|
||||
// live file; renamed generations are reclaimed by the ownership sweep,
|
||||
// since a receipt write can succeed before reporting a durability failure.
|
||||
if (stagingReleased)
|
||||
for (const suffix of ['', '.wal', '.shadow']) await fs.rm(staging + suffix, { force: true });
|
||||
}
|
||||
}
|
||||
388
gitnexus/src/core/lbug/legacy-vector-source.ts
Normal file
388
gitnexus/src/core/lbug/legacy-vector-source.ts
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
/** Bounded, read-only reuse of verified legacy monolithic embedding groups. */
|
||||
import lbug, { type LbugValue } from '@ladybugdb/core';
|
||||
import fs from 'node:fs/promises';
|
||||
import {
|
||||
openLbugConnection,
|
||||
closeLbugConnection,
|
||||
type LbugConnectionHandle,
|
||||
} from './lbug-config.js';
|
||||
import {
|
||||
VectorStore,
|
||||
type LegacyVectorGroup,
|
||||
type VectorNodeReceipt,
|
||||
type StoredVectorChunk,
|
||||
} from './vector-store.js';
|
||||
import { assertValidGraphNodeId } from '../graph/node-identity.js';
|
||||
import type { EmbeddingIdentity } from '../embeddings/embedding-identity.js';
|
||||
import { resolveEmbeddingConfig } from '../embeddings/config.js';
|
||||
import {
|
||||
iterateEmbeddingInputs,
|
||||
type EmbeddingReuseOptions,
|
||||
} from '../embeddings/embedding-inputs.js';
|
||||
import { isShortLabel, type EmbeddableNode, type EmbeddingConfig } from '../embeddings/types.js';
|
||||
|
||||
export interface LegacyVectorSourceOptions {
|
||||
/** Caller validates ownership and holds the slot lock. WAL recovery and
|
||||
* checkpointing must already have completed in the graph process. */
|
||||
path: string;
|
||||
sourceIdentity: string;
|
||||
recordedIdentity?: Partial<EmbeddingIdentity>;
|
||||
expectedIdentity: EmbeddingIdentity;
|
||||
/** Optional historical input receipt; never synthesized from current config. */
|
||||
recordedInputIdentity?: string;
|
||||
expectedInputIdentity: string;
|
||||
config?: Partial<EmbeddingConfig>;
|
||||
bufferPoolBytes: number;
|
||||
threads: number;
|
||||
batchBytes: number;
|
||||
signal?: AbortSignal;
|
||||
log?: (message: string) => void;
|
||||
}
|
||||
|
||||
/** A row hash does not identify its provider. Never fill missing historical
|
||||
* fields from today's configuration, including for a mere file relocation. */
|
||||
export function legacyVectorCompatibility(
|
||||
recorded: Partial<EmbeddingIdentity> | undefined,
|
||||
expected: EmbeddingIdentity,
|
||||
): string | undefined {
|
||||
if (
|
||||
!recorded ||
|
||||
typeof recorded.provider !== 'string' ||
|
||||
!/^(local|http:[a-f0-9]{64})$/.test(recorded.provider) ||
|
||||
typeof recorded.model !== 'string' ||
|
||||
!recorded.model ||
|
||||
!Number.isSafeInteger(recorded.dimensions) ||
|
||||
!recorded.dimensions ||
|
||||
recorded.dimensions < 1
|
||||
)
|
||||
return 'Legacy embedding provider/model/dimensions provenance is unknown; source retained.';
|
||||
if (
|
||||
recorded.provider !== expected.provider ||
|
||||
recorded.model !== expected.model ||
|
||||
recorded.dimensions !== expected.dimensions
|
||||
)
|
||||
return 'Legacy embedding provider/model/dimensions differ from this request; source retained.';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const count = (value: unknown): number => {
|
||||
if (typeof value !== 'number' && typeof value !== 'bigint')
|
||||
throw new Error('Invalid legacy vector count');
|
||||
const n = Number(value);
|
||||
if (!Number.isSafeInteger(n) || n < 0) throw new Error('Invalid legacy vector count');
|
||||
return n;
|
||||
};
|
||||
type Row = Record<string, unknown>;
|
||||
function single(result: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult {
|
||||
if (!Array.isArray(result)) return result;
|
||||
if (result.length === 1) return result[0];
|
||||
for (const part of result) part.close();
|
||||
throw new Error('Unexpected legacy query result count');
|
||||
}
|
||||
|
||||
export class LegacyVectorSource {
|
||||
readonly requiresInputValidation = true;
|
||||
private readonly statements = new Map<string, lbug.PreparedStatement>();
|
||||
private readonly config: EmbeddingConfig;
|
||||
private active = false;
|
||||
private closed = false;
|
||||
private closing?: Promise<void>;
|
||||
private constructor(
|
||||
readonly options: LegacyVectorSourceOptions,
|
||||
private readonly handle: LbugConnectionHandle,
|
||||
private readonly index: VectorStore,
|
||||
) {
|
||||
this.config = resolveEmbeddingConfig(options.config);
|
||||
}
|
||||
|
||||
static async open(
|
||||
options: LegacyVectorSourceOptions,
|
||||
index: VectorStore,
|
||||
excluded: () => AsyncIterable<string> | Iterable<string>,
|
||||
): Promise<LegacyVectorSource> {
|
||||
const incompatible = legacyVectorCompatibility(
|
||||
options.recordedIdentity,
|
||||
options.expectedIdentity,
|
||||
);
|
||||
if (incompatible) throw new Error(incompatible);
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(options.sourceIdentity) ||
|
||||
!/^[a-f0-9]{64}$/.test(options.expectedInputIdentity) ||
|
||||
options.expectedIdentity.dimensions !== index.options.dimensions ||
|
||||
!Number.isSafeInteger(options.batchBytes) ||
|
||||
options.batchBytes < 1024
|
||||
)
|
||||
throw new Error('Invalid legacy vector reuse reservation');
|
||||
if (
|
||||
options.recordedInputIdentity !== undefined &&
|
||||
options.recordedInputIdentity !== options.expectedInputIdentity
|
||||
)
|
||||
throw new Error(
|
||||
'Legacy embedding input provenance differs from this request; source retained.',
|
||||
);
|
||||
if (!options.recordedInputIdentity)
|
||||
options.log?.(
|
||||
'Legacy multi-chunk input provenance is unknown (chunk size/overlap); source retained and those groups require regeneration.',
|
||||
);
|
||||
options.signal?.throwIfAborted();
|
||||
const stat = await fs.lstat(options.path);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Legacy vectors require an owned regular database');
|
||||
const source = new LegacyVectorSource(
|
||||
options,
|
||||
await openLbugConnection(lbug, options.path, {
|
||||
readOnly: true,
|
||||
bufferPoolBytes: options.bufferPoolBytes,
|
||||
threads: options.threads,
|
||||
}),
|
||||
index,
|
||||
);
|
||||
try {
|
||||
await index.prepareLegacyReuseIndex(
|
||||
options.sourceIdentity,
|
||||
() => source.groups(),
|
||||
excluded,
|
||||
options.signal,
|
||||
);
|
||||
return source;
|
||||
} catch (error) {
|
||||
await source.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private acquire(): void {
|
||||
if (this.active || this.closed) throw new Error('Legacy vector source is closed or active');
|
||||
this.options.signal?.throwIfAborted();
|
||||
this.active = true;
|
||||
}
|
||||
private async result(sql: string, params?: Record<string, LbugValue>): Promise<lbug.QueryResult> {
|
||||
if (!params) return single(await this.handle.conn.query(sql));
|
||||
let prepared = this.statements.get(sql);
|
||||
if (!prepared) {
|
||||
prepared = await this.handle.conn.prepare(sql);
|
||||
if (!prepared.isSuccess()) throw new Error(prepared.getErrorMessage());
|
||||
this.statements.set(sql, prepared);
|
||||
}
|
||||
return single(await this.handle.conn.execute(prepared, params));
|
||||
}
|
||||
|
||||
private async *groups(): AsyncGenerator<LegacyVectorGroup> {
|
||||
this.acquire();
|
||||
let result: lbug.QueryResult | undefined;
|
||||
try {
|
||||
// Native sorting is charged to this explicit pool; JS retains one group
|
||||
// and never fetches vector payloads while building the receipt index.
|
||||
result = await this.result(
|
||||
'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS contentHash, e.startLine AS startLine, e.endLine AS endLine ORDER BY e.nodeId, e.chunkIndex',
|
||||
);
|
||||
let group: LegacyVectorGroup | undefined;
|
||||
let valid = true;
|
||||
while (result.hasNext()) {
|
||||
this.options.signal?.throwIfAborted();
|
||||
const row = result.getNextSync();
|
||||
if (typeof row.nodeId !== 'string') throw new Error('Invalid legacy node identity');
|
||||
assertValidGraphNodeId(row.nodeId, 'Legacy vector source');
|
||||
if (group?.nodeId !== row.nodeId) {
|
||||
if (group && valid) yield group;
|
||||
group = {
|
||||
nodeId: row.nodeId,
|
||||
contentHash: typeof row.contentHash === 'string' ? row.contentHash : '',
|
||||
chunks: 0,
|
||||
sourceStartLine: count(row.startLine),
|
||||
sourceEndLine: count(row.endLine),
|
||||
};
|
||||
valid = Boolean(group.contentHash);
|
||||
}
|
||||
if (!group) throw new Error('Missing legacy group');
|
||||
const ordinal = count(row.chunkIndex),
|
||||
start = count(row.startLine),
|
||||
end = count(row.endLine);
|
||||
if (
|
||||
ordinal !== group.chunks ||
|
||||
row.id !== `${group.nodeId}:${ordinal}` ||
|
||||
row.contentHash !== group.contentHash ||
|
||||
end < start
|
||||
)
|
||||
valid = false;
|
||||
group.chunks++;
|
||||
group.sourceEndLine = Math.max(group.sourceEndLine, end);
|
||||
}
|
||||
if (group && valid) yield group;
|
||||
} finally {
|
||||
try {
|
||||
result?.close();
|
||||
} finally {
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async rows(
|
||||
receipt: VectorNodeReceipt,
|
||||
start: number,
|
||||
length: number,
|
||||
vectors: boolean,
|
||||
): Promise<Row[]> {
|
||||
const result = await this.result(
|
||||
`UNWIND $ids AS id MATCH (e:CodeEmbedding {id:id}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.contentHash AS contentHash${vectors ? ', e.embedding AS embedding' : ''}`,
|
||||
{
|
||||
ids: Array.from({ length }, (_, i) => `${receipt.nodeId}:${start + i}`),
|
||||
},
|
||||
);
|
||||
const rows: Row[] = [];
|
||||
try {
|
||||
while (result.hasNext()) {
|
||||
if (rows.length >= length) throw new Error('Legacy query exceeded its bounded result size');
|
||||
rows.push(result.getNextSync());
|
||||
}
|
||||
} finally {
|
||||
result.close();
|
||||
}
|
||||
rows.sort((a, b) => count(a.chunkIndex) - count(b.chunkIndex));
|
||||
if (
|
||||
rows.length !== length ||
|
||||
rows.some(
|
||||
(row, i) =>
|
||||
row.nodeId !== receipt.nodeId ||
|
||||
row.contentHash !== receipt.contentHash ||
|
||||
count(row.chunkIndex) !== start + i,
|
||||
)
|
||||
)
|
||||
throw new Error('Legacy complete group changed; source retained');
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Legacy hashes did not record chunk-size/overlap. Multiple chunks may have
|
||||
* identical line spans but different character boundaries, so their layout
|
||||
* cannot replace an input receipt. Short labels ignore those settings; a
|
||||
* complete single-chunk group covers its full input without overlap. */
|
||||
async reusable(
|
||||
nodeId: string,
|
||||
contentHash: string,
|
||||
options: EmbeddingReuseOptions,
|
||||
): Promise<VectorNodeReceipt | null> {
|
||||
const { node, prepareInputValidation, inputs, matchesHash } = options;
|
||||
this.acquire();
|
||||
const originalMethods = node.methodNames,
|
||||
originalFields = node.fieldNames;
|
||||
try {
|
||||
const group = await this.index.legacyReuseGroup(this.options.sourceIdentity, nodeId);
|
||||
if (!group || node.id !== nodeId) return null;
|
||||
const sourceAware = /^[a-f0-9]{40}:[a-f0-9]{40}$/.test(group.contentHash);
|
||||
if (group.contentHash !== contentHash && !(await matchesHash?.(group.contentHash)))
|
||||
return null;
|
||||
if (
|
||||
!this.options.recordedInputIdentity &&
|
||||
!sourceAware &&
|
||||
!isShortLabel(node.label) &&
|
||||
group.chunks > 1
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
await prepareInputValidation?.();
|
||||
const receipt: VectorNodeReceipt = { ...group, complete: true };
|
||||
const rowBytes = 512 + 8 * nodeId.length;
|
||||
if (rowBytes > this.options.batchBytes)
|
||||
throw new Error('Legacy metadata does not fit its reservation');
|
||||
const blockSize = Math.min(
|
||||
64,
|
||||
Math.max(1, Math.floor(Math.min(32768, this.options.batchBytes) / rowBytes)),
|
||||
);
|
||||
let block: Row[] = [],
|
||||
ordinal = 0,
|
||||
delta: number | undefined;
|
||||
for await (const span of inputs ? inputs() : this.inputSpans(node)) {
|
||||
if (ordinal >= group.chunks) return null;
|
||||
if (ordinal % blockSize === 0)
|
||||
block = await this.rows(
|
||||
receipt,
|
||||
ordinal,
|
||||
Math.min(blockSize, group.chunks - ordinal),
|
||||
false,
|
||||
);
|
||||
const row = block[ordinal % blockSize];
|
||||
const start = count(row.startLine),
|
||||
end = count(row.endLine);
|
||||
delta ??= span.startLine - start;
|
||||
if (start + delta !== span.startLine || end + delta !== span.endLine) return null;
|
||||
ordinal++;
|
||||
}
|
||||
if (ordinal !== group.chunks || delta === undefined || (node.startLine ?? 0) - delta < 0)
|
||||
return null;
|
||||
// Legacy rows recorded chunk ranges, not the original node's end.
|
||||
// Do not let the copy path clamp the final chunk to an invented range.
|
||||
return {
|
||||
...receipt,
|
||||
sourceStartLine: (node.startLine ?? 0) - delta,
|
||||
sourceEndLine: undefined,
|
||||
};
|
||||
} finally {
|
||||
node.methodNames = originalMethods;
|
||||
node.fieldNames = originalFields;
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async *inputSpans(
|
||||
node: EmbeddableNode,
|
||||
): AsyncGenerator<{ startLine: number; endLine: number }> {
|
||||
yield* iterateEmbeddingInputs(node, this.config, { signal: this.options.signal });
|
||||
}
|
||||
|
||||
async readChunks(
|
||||
receipt: VectorNodeReceipt,
|
||||
start: number,
|
||||
length: number,
|
||||
): Promise<StoredVectorChunk[]> {
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
start < 0 ||
|
||||
!Number.isSafeInteger(length) ||
|
||||
length < 1 ||
|
||||
length > 256 ||
|
||||
start + length > receipt.chunks ||
|
||||
length * (32 * this.options.expectedIdentity.dimensions + 1024 + 4 * receipt.nodeId.length) >
|
||||
this.options.batchBytes
|
||||
)
|
||||
throw new Error('Invalid legacy vector read batch');
|
||||
this.acquire();
|
||||
try {
|
||||
return (await this.rows(receipt, start, length, true)).map((row) => {
|
||||
if (
|
||||
!Array.isArray(row.embedding) ||
|
||||
row.embedding.length !== this.options.expectedIdentity.dimensions ||
|
||||
row.embedding.some((value) => typeof value !== 'number' || !Number.isFinite(value))
|
||||
)
|
||||
throw new Error('Invalid legacy vector dimensions or values');
|
||||
const startLine = count(row.startLine),
|
||||
endLine = count(row.endLine);
|
||||
if (endLine < startLine) throw new Error('Invalid legacy vector lines');
|
||||
return {
|
||||
chunkIndex: count(row.chunkIndex),
|
||||
startLine,
|
||||
endLine,
|
||||
embedding: row.embedding as number[],
|
||||
};
|
||||
});
|
||||
} finally {
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
if (this.closing) return this.closing;
|
||||
if (this.active) throw new Error('Cannot close an active legacy vector read');
|
||||
this.closed = true;
|
||||
this.statements.clear();
|
||||
this.closing = closeLbugConnection(this.handle, {
|
||||
strict: true,
|
||||
databasePath: this.options.path,
|
||||
});
|
||||
try {
|
||||
await this.closing;
|
||||
} catch (error) {
|
||||
this.closing = undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -60,6 +60,9 @@ interface PoolEntry {
|
|||
}>;
|
||||
lastUsed: number;
|
||||
dbPath: string;
|
||||
/** Immutable vector hard links share one native pool, independently of the
|
||||
* path each checkout retains. Graph and injected handles remain path-keyed. */
|
||||
dbCacheKey?: string;
|
||||
/** Filesystem identity of the on-disk DB at open time. When `analyze`
|
||||
* rebuilds or mutates the index, this diverges from the current file and
|
||||
* initLbug re-opens the pool onto the new file instead of serving the
|
||||
|
|
@ -68,6 +71,14 @@ interface PoolEntry {
|
|||
dbIdentity: DbIdentity | null;
|
||||
/** Set to true when the pool entry is closed — checkin will close orphaned connections */
|
||||
closed: boolean;
|
||||
artifactOptions?: VectorPoolOptions;
|
||||
}
|
||||
|
||||
/** Separate vector files never inherit the graph's default native pool. */
|
||||
export interface VectorPoolOptions {
|
||||
kind: 'vector';
|
||||
bufferPoolBytes: number;
|
||||
threads: number;
|
||||
}
|
||||
|
||||
/** Filesystem identity used to detect an index rebuilt/mutated under a live
|
||||
|
|
@ -131,9 +142,8 @@ export function addPoolCloseListener(listener: PoolCloseListener): () => void {
|
|||
}
|
||||
|
||||
/**
|
||||
* Shared Database cache keyed by resolved dbPath.
|
||||
* Multiple repoIds pointing to the same path share one native Database
|
||||
* object to avoid exhausting the buffer manager's mmap budget.
|
||||
* Shared Database cache keyed by graph path or immutable vector file identity.
|
||||
* Sibling hard links must not multiply a sealed artifact's native reservation.
|
||||
*/
|
||||
interface SharedDB {
|
||||
db: lbug.Database;
|
||||
|
|
@ -157,6 +167,7 @@ interface SharedDB {
|
|||
dbIdentity?: DbIdentity | null;
|
||||
/** When true, closeOne skips db.close() — the Database is owned externally. */
|
||||
external?: boolean;
|
||||
artifactOptions?: VectorPoolOptions;
|
||||
}
|
||||
const dbCache = new Map<string, SharedDB>();
|
||||
|
||||
|
|
@ -348,12 +359,17 @@ export const getMaxResidentRepos = (): number => MAX_POOL_SIZE;
|
|||
* entry is pinned, no eviction occurs and the pool transiently exceeds
|
||||
* MAX_POOL_SIZE (see the pinnedRepos docstring).
|
||||
*/
|
||||
async function evictLRU(): Promise<void> {
|
||||
if (pool.size < MAX_POOL_SIZE) return;
|
||||
async function evictLRU(kind?: 'vector'): Promise<void> {
|
||||
// A vector artifact must not halve the existing graph-cache capacity. Each
|
||||
// lane has its own bounded allowance; vector handles reserve only 64 MiB.
|
||||
let residents = 0;
|
||||
for (const entry of pool.values()) if (entry.artifactOptions?.kind === kind) residents++;
|
||||
if (residents < MAX_POOL_SIZE) return;
|
||||
|
||||
let oldestId: string | null = null;
|
||||
let oldestTime = Infinity;
|
||||
for (const [id, entry] of pool) {
|
||||
if (entry.artifactOptions?.kind !== kind) continue;
|
||||
if (pinnedRepos.has(id)) continue;
|
||||
if (entry.checkedOut === 0 && entry.lastUsed < oldestTime) {
|
||||
oldestTime = entry.lastUsed;
|
||||
|
|
@ -431,7 +447,8 @@ async function closeOne(repoId: string): Promise<void> {
|
|||
// Only close the Database when no other repoIds reference it.
|
||||
// External databases (injected via initLbugWithDb) are never closed here —
|
||||
// the core adapter owns them and handles their lifecycle.
|
||||
const shared = dbCache.get(entry.dbPath);
|
||||
const cacheKey = entry.dbCacheKey ?? entry.dbPath;
|
||||
const shared = dbCache.get(cacheKey);
|
||||
if (shared) {
|
||||
shared.refCount--;
|
||||
if (shared.refCount === 0) {
|
||||
|
|
@ -449,7 +466,7 @@ async function closeOne(repoId: string): Promise<void> {
|
|||
// subsequent reopen (evictLRU / the "idle & changed" path below) must not
|
||||
// race. See the awaited call site in evictLRU for the full rationale.
|
||||
await shared.db.close().catch(() => {});
|
||||
dbCache.delete(entry.dbPath);
|
||||
dbCache.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -783,10 +800,24 @@ function ladybugUnavailableError(repoId: string, err: Error | undefined): Error
|
|||
* bookkeeping on "did the pool actually roll over" (LocalBackend) use the
|
||||
* return value; callers that only need the pool ready can ignore it.
|
||||
*/
|
||||
export const initLbug = async (repoId: string, dbPath: string): Promise<boolean> => {
|
||||
export const initLbug = async (
|
||||
repoId: string,
|
||||
dbPath: string,
|
||||
artifactOptions?: VectorPoolOptions,
|
||||
): Promise<boolean> => {
|
||||
if (
|
||||
artifactOptions &&
|
||||
(artifactOptions.kind !== 'vector' ||
|
||||
!Number.isSafeInteger(artifactOptions.bufferPoolBytes) ||
|
||||
artifactOptions.bufferPoolBytes < 8 * 1024 * 1024 ||
|
||||
!Number.isSafeInteger(artifactOptions.threads) ||
|
||||
artifactOptions.threads < 1)
|
||||
)
|
||||
throw new Error('Invalid vector reader resource reservation');
|
||||
const reservation = artifactOptions ? Object.freeze({ ...artifactOptions }) : undefined;
|
||||
let lastError: Error | undefined;
|
||||
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
|
||||
const result = await withPoolLock(() => initLbugInner(repoId, dbPath));
|
||||
const result = await withPoolLock(() => initLbugInner(repoId, dbPath, reservation));
|
||||
if (result.status === 'done') return result.reopened;
|
||||
lastError = result.error;
|
||||
if (attempt === LOCK_RETRY_ATTEMPTS) break;
|
||||
|
|
@ -795,7 +826,11 @@ export const initLbug = async (repoId: string, dbPath: string): Promise<boolean>
|
|||
throw ladybugUnavailableError(repoId, lastError);
|
||||
};
|
||||
|
||||
const initLbugInner = async (repoId: string, dbPath: string): Promise<InitLbugAttempt> => {
|
||||
const initLbugInner = async (
|
||||
repoId: string,
|
||||
dbPath: string,
|
||||
artifactOptions?: VectorPoolOptions,
|
||||
): Promise<InitLbugAttempt> => {
|
||||
const existing = pool.get(repoId);
|
||||
if (existing) {
|
||||
existing.lastUsed = Date.now();
|
||||
|
|
@ -804,7 +839,12 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise<InitLbugAt
|
|||
// unlinked-but-open) inode until LRU/idle eviction — a stale-read window
|
||||
// of up to IDLE_TIMEOUT_MS after analyze finishes.
|
||||
const current = await statDbIdentity(dbPath);
|
||||
if (!dbIdentityChanged(existing.dbIdentity, current)) {
|
||||
if (
|
||||
existing.dbPath === dbPath &&
|
||||
!sameArtifactOptions(existing.artifactOptions, artifactOptions)
|
||||
)
|
||||
throw new Error('An open database has a different resource reservation');
|
||||
if (existing.dbPath === dbPath && !dbIdentityChanged(existing.dbIdentity, current)) {
|
||||
return { status: 'done', reopened: false }; // unchanged → reuse
|
||||
}
|
||||
// A query is in flight on this entry; closing its connection (and the
|
||||
|
|
@ -818,27 +858,92 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise<InitLbugAt
|
|||
// freshness THROUGH initLbug (rather than calling closeLbug directly) get
|
||||
// this guard for free; that is why LocalBackend delegates here (#2614).
|
||||
if (existing.checkedOut > 0) return { status: 'done', reopened: false };
|
||||
// Closing only this consumer would leave the shared native handle alive
|
||||
// For graph paths, closing only this consumer leaves the shared handle alive
|
||||
// and doInitLbug would attach eight new connections to that same old
|
||||
// generation. Keep this pool intact until the other consumers release it;
|
||||
// its opened identity stays divergent so the next init can retry then.
|
||||
const shared = dbCache.get(existing.dbPath);
|
||||
if (shared && shared.db === existing.db && shared.refCount > 1) {
|
||||
const shared = dbCache.get(existing.dbCacheKey ?? existing.dbPath);
|
||||
if (
|
||||
!artifactOptions &&
|
||||
existing.dbPath === dbPath &&
|
||||
shared &&
|
||||
shared.db === existing.db &&
|
||||
shared.refCount > 1
|
||||
) {
|
||||
return { status: 'done', reopened: false };
|
||||
}
|
||||
// Awaited: see the rationale on the evictLRU call site in doInitLbug below.
|
||||
await closeOne(repoId); // idle & changed → evict, then fall through to reopen the new file
|
||||
}
|
||||
|
||||
return doInitLbug(repoId, dbPath);
|
||||
return doInitLbug(repoId, dbPath, artifactOptions);
|
||||
};
|
||||
|
||||
function sameArtifactOptions(a?: VectorPoolOptions, b?: VectorPoolOptions): boolean {
|
||||
return (
|
||||
a?.kind === b?.kind && a?.bufferPoolBytes === b?.bufferPoolBytes && a?.threads === b?.threads
|
||||
);
|
||||
}
|
||||
|
||||
/** Validate every retained path before sharing a native handle. No content
|
||||
* hash or copy: sealed files can be pinned by hard links without extra pools.
|
||||
* An unavailable inode identity falls back to the existing path isolation. */
|
||||
async function vectorDatabaseKey(dbPath: string): Promise<string> {
|
||||
const stat = await fs.lstat(dbPath, { bigint: true });
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Vector artifact must be a regular file');
|
||||
for (const suffix of ['.wal', '.shadow']) {
|
||||
try {
|
||||
const sidecar = await fs.lstat(dbPath + suffix);
|
||||
if (sidecar.size || !sidecar.isFile() || sidecar.isSymbolicLink())
|
||||
throw new Error('Published vector artifact has unfinished native writes');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
// NUL cannot occur in a filesystem path, so these keys cannot collide with
|
||||
// the graph cache. ctime changes when another hard link is added; mtime does not.
|
||||
return typeof stat.ino === 'bigint' &&
|
||||
stat.ino > 0n &&
|
||||
typeof stat.dev === 'bigint' &&
|
||||
stat.dev > 0n
|
||||
? `vector\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}`
|
||||
: `vector\0path\0${dbPath}`;
|
||||
}
|
||||
|
||||
/** Called after vectorDatabaseKey validates the immutable file and sidecars.
|
||||
* Readers never quarantine, replay, or repair their writer's sidecars. */
|
||||
async function openVectorReadOnlyDatabase(
|
||||
dbPath: string,
|
||||
options: VectorPoolOptions,
|
||||
): Promise<lbug.Database> {
|
||||
silenceStdout();
|
||||
let db: lbug.Database | undefined;
|
||||
try {
|
||||
db = createLbugDatabase(lbug, toNativeSafePath(dbPath), {
|
||||
readOnly: true,
|
||||
bufferPoolBytes: options.bufferPoolBytes,
|
||||
});
|
||||
await db.init();
|
||||
return db;
|
||||
} catch (error) {
|
||||
if (db) await db.close().catch(() => {});
|
||||
throw error;
|
||||
} finally {
|
||||
restoreStdout();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal init — creates DB, pre-warms connections, loads FTS, then registers pool.
|
||||
* Pool entry is registered LAST so concurrent executeQuery calls see either
|
||||
* "not initialized" (and throw) or a fully ready pool — never a half-built one.
|
||||
*/
|
||||
async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttempt> {
|
||||
async function doInitLbug(
|
||||
repoId: string,
|
||||
dbPath: string,
|
||||
artifactOptions?: VectorPoolOptions,
|
||||
): Promise<InitLbugAttempt> {
|
||||
// Check if database exists
|
||||
try {
|
||||
await fs.stat(dbPath);
|
||||
|
|
@ -854,11 +959,14 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
// evictLRU is a no-op unless the pool is full again (another repo may have
|
||||
// taken the slot we freed on a prior attempt). Skipping it on retry would
|
||||
// let a 6th native open race a still-resident victim's checkpoint.
|
||||
await evictLRU();
|
||||
await evictLRU(artifactOptions?.kind);
|
||||
|
||||
// Reuse an existing native Database if another repoId already opened this path.
|
||||
// This prevents buffer manager exhaustion from multiple mmap regions on the same file.
|
||||
let shared = dbCache.get(dbPath);
|
||||
// Immutable vector aliases share the reservation even when their checkout
|
||||
// paths differ. Graph databases retain their existing path-based lifecycle.
|
||||
const cacheKey = artifactOptions ? await vectorDatabaseKey(dbPath) : dbPath;
|
||||
let shared = dbCache.get(cacheKey);
|
||||
if (shared && !sameArtifactOptions(shared.artifactOptions, artifactOptions))
|
||||
throw new Error('A shared database has a different resource reservation');
|
||||
if (shared && !shared.external && shared.dbIdentity) {
|
||||
// #2614 F2: a cached read-only Database is keyed by dbPath and shared across
|
||||
// pool consumers. If the on-disk index was rebuilt/swapped (new inode) while
|
||||
|
|
@ -882,11 +990,20 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
// avoids lock conflicts when `gitnexus analyze` is writing. This attempt
|
||||
// is one native open; lock-retry backoff lives in initLbug.
|
||||
try {
|
||||
const db = await openReadOnlyDatabase(dbPath);
|
||||
shared = { db, refCount: 0, ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath) };
|
||||
dbCache.set(dbPath, shared);
|
||||
const db = artifactOptions
|
||||
? await openVectorReadOnlyDatabase(dbPath, artifactOptions)
|
||||
: await openReadOnlyDatabase(dbPath);
|
||||
shared = {
|
||||
db,
|
||||
refCount: 0,
|
||||
ftsLoaded: false,
|
||||
dbIdentity: await statDbIdentity(dbPath),
|
||||
artifactOptions,
|
||||
};
|
||||
dbCache.set(cacheKey, shared);
|
||||
} catch (err: unknown) {
|
||||
const lastError = err instanceof Error ? err : new Error(String(err));
|
||||
if (artifactOptions) throw lastError;
|
||||
|
||||
// Not retryable: the on-disk file's storage version doesn't change
|
||||
// on its own. Fail immediately with an actionable message.
|
||||
|
|
@ -904,7 +1021,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
ftsLoaded: false,
|
||||
dbIdentity: await statDbIdentity(dbPath),
|
||||
};
|
||||
dbCache.set(dbPath, shared);
|
||||
dbCache.set(cacheKey, shared);
|
||||
} catch (retryErr) {
|
||||
throw new Error(
|
||||
`LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` +
|
||||
|
|
@ -943,8 +1060,18 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
const available: lbug.Connection[] = [];
|
||||
try {
|
||||
for (let i = 0; i < MAX_CONNS_PER_REPO; i++) {
|
||||
available.push(createConnection(db));
|
||||
const connection = createConnection(db);
|
||||
available.push(connection);
|
||||
if (artifactOptions) await connection.setMaxNumThreadForExec(artifactOptions.threads);
|
||||
}
|
||||
} catch (error) {
|
||||
await Promise.allSettled(available.map((connection) => connection.close()));
|
||||
shared.refCount--;
|
||||
if (shared.refCount === 0) {
|
||||
dbCache.delete(cacheKey);
|
||||
if (!shared.external) await db.close().catch(() => {});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
preWarmActive = false;
|
||||
}
|
||||
|
|
@ -955,7 +1082,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
// policy: 'load-only' — the read pool must never trigger a network
|
||||
// install; analyze owns extension installation. If LOAD fails, search
|
||||
// features degrade gracefully and the user-facing query path proceeds.
|
||||
if (!shared.ftsLoaded) {
|
||||
if (!artifactOptions && !shared.ftsLoaded) {
|
||||
shared.ftsLoaded = await loadFTSExtension(available[0], { policy: 'load-only' });
|
||||
}
|
||||
// Register pool entry only after all connections are pre-warmed and FTS is
|
||||
|
|
@ -971,8 +1098,10 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttem
|
|||
waiters: [],
|
||||
lastUsed: Date.now(),
|
||||
dbPath,
|
||||
dbCacheKey: cacheKey,
|
||||
dbIdentity,
|
||||
closed: false,
|
||||
artifactOptions,
|
||||
});
|
||||
ensureIdleTimer();
|
||||
traceRss('init', repoId);
|
||||
|
|
@ -1065,7 +1194,7 @@ export async function ensureVectorExtension(repoId: string): Promise<boolean> {
|
|||
throw new Error(`LadybugDB not initialized for repo "${repoId}". Call initLbug first.`);
|
||||
}
|
||||
|
||||
const shared = dbCache.get(entry.dbPath);
|
||||
const shared = dbCache.get(entry.dbCacheKey ?? entry.dbPath);
|
||||
if (!shared) {
|
||||
throw new Error(`LadybugDB shared handle is unavailable for repo "${repoId}".`);
|
||||
}
|
||||
|
|
|
|||
1026
gitnexus/src/core/lbug/vector-store.ts
Normal file
1026
gitnexus/src/core/lbug/vector-store.ts
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -37,10 +37,13 @@ import {
|
|||
type RegistryEntry,
|
||||
} from '../storage/repo-manager.js';
|
||||
import { isMissingFilesystemError, loadMeta, type RepoMeta } from '../storage/repo-meta.js';
|
||||
import { retainSharedVectorArtifact } from '../storage/shared-vector-artifact.js';
|
||||
import { selectVectorGeneration } from '../storage/vector-generation.js';
|
||||
import {
|
||||
cloneStoreKey,
|
||||
commitGraphDir,
|
||||
resolveGraphPath,
|
||||
resolveGraphWritePath,
|
||||
resolveSharedStore,
|
||||
sharedStoreLayout,
|
||||
storeRootOfCheckoutSlot,
|
||||
|
|
@ -58,6 +61,7 @@ import {
|
|||
import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from '../storage/storage-constants.js';
|
||||
import { wipeLbugDbFiles } from './lbug/lbug-adapter.js';
|
||||
import { inspectLbugSidecars } from './lbug/sidecar-recovery.js';
|
||||
import { getExactScanLimit } from './platform/capabilities.js';
|
||||
|
||||
type Log = (msg: string) => void;
|
||||
|
||||
|
|
@ -103,11 +107,117 @@ export const featureKeyOf = (meta: RepoMeta): string => {
|
|||
for (const [k, v] of Object.entries(meta)) {
|
||||
if (!FEATURE_KEY_EXCLUDED.has(k)) kept[k] = v;
|
||||
}
|
||||
// Embedding presence changes graph content but lives only in `stats`.
|
||||
kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0;
|
||||
if (hasSeparateGraphGeneration(meta)) {
|
||||
// Logical IDs and separate vector work do not change the graph's facts.
|
||||
// Keep all unknown generation/capability fields in the denylist hash.
|
||||
const generation: Record<string, unknown> = { ...meta.graphGeneration };
|
||||
delete generation.id;
|
||||
kept.graphGeneration = generation;
|
||||
for (const key of ['embeddingJob', 'vectorGeneration', 'legacyVectorSource', 'embeddingDims'])
|
||||
delete kept[key];
|
||||
if (meta.capabilities) {
|
||||
const capabilities: Record<string, unknown> = { ...meta.capabilities };
|
||||
delete capabilities.vectorSearch;
|
||||
kept.capabilities = capabilities;
|
||||
}
|
||||
} else {
|
||||
// Legacy embeddings still live inside the graph database.
|
||||
kept.hasEmbeddings = (meta.stats?.embeddings ?? 0) > 0;
|
||||
}
|
||||
return createHash('sha256').update(stableStringify(kept)).digest('hex').slice(0, 16);
|
||||
};
|
||||
|
||||
function hasSeparateGraphGeneration(meta: RepoMeta): boolean {
|
||||
const generation = meta.graphGeneration;
|
||||
return (
|
||||
generation?.version === 1 &&
|
||||
/^[a-f0-9]{32}$/.test(generation.id) &&
|
||||
/^[a-f0-9]{64}$/.test(generation.coveredContent)
|
||||
);
|
||||
}
|
||||
|
||||
/** Share completed immutable files only. The original slot keeps its mutable
|
||||
* job and receipts; siblings never inherit ownership of that paid work. */
|
||||
async function retainSharedMetadata(
|
||||
source: string,
|
||||
destination: string,
|
||||
meta: RepoMeta,
|
||||
allowCopy = false,
|
||||
): Promise<RepoMeta> {
|
||||
if (
|
||||
!meta.graphGeneration &&
|
||||
!meta.embeddingJob &&
|
||||
!meta.vectorGeneration &&
|
||||
!meta.legacyVectorSource
|
||||
)
|
||||
return { ...meta };
|
||||
if (!hasSeparateGraphGeneration(meta))
|
||||
throw new Error('Cannot share an unverified logical graph generation');
|
||||
const exported = { ...meta };
|
||||
delete exported.embeddingJob;
|
||||
delete exported.embeddingCheckpoint;
|
||||
delete exported.legacyVectorSource;
|
||||
const selected = selectVectorGeneration(
|
||||
source,
|
||||
exported,
|
||||
resolveGraphPath(source),
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
const vector = meta.vectorGeneration;
|
||||
const active = meta.embeddingJob?.artifactIdentity ?? meta.embeddingJob?.identity;
|
||||
const completed =
|
||||
vector &&
|
||||
active !== (vector.artifactIdentity ?? vector.identity) &&
|
||||
(selected.kind === 'ready' ||
|
||||
(selected.kind === 'unavailable' && selected.reason === 'embeddings-empty'));
|
||||
if (completed) {
|
||||
await retainSharedVectorArtifact(source, destination, vector, allowCopy);
|
||||
} else {
|
||||
delete exported.vectorGeneration;
|
||||
delete exported.embeddingDims;
|
||||
}
|
||||
exported.stats = { ...meta.stats, embeddings: completed ? vector.chunks : 0 };
|
||||
if (meta.capabilities) {
|
||||
const exactScanLimit = meta.capabilities.vectorSearch?.exactScanLimit ?? getExactScanLimit();
|
||||
const readable =
|
||||
completed &&
|
||||
vector.chunks > 0 &&
|
||||
(vector.indexKind === 'hnsw' || vector.chunks <= exactScanLimit);
|
||||
exported.capabilities = {
|
||||
...meta.capabilities,
|
||||
vectorSearch: {
|
||||
provider: completed && vector.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan',
|
||||
status: readable
|
||||
? vector.indexKind === 'hnsw'
|
||||
? 'vector-index'
|
||||
: 'exact-scan'
|
||||
: 'unavailable',
|
||||
exactScanLimit,
|
||||
coverage: completed
|
||||
? vector.graphGeneration === meta.graphGeneration?.id
|
||||
? 'current'
|
||||
: 'preserved'
|
||||
: 'unknown',
|
||||
...(!readable
|
||||
? {
|
||||
reason:
|
||||
completed && vector.chunks ? 'vector-index-pending' : 'embeddings-unavailable',
|
||||
}
|
||||
: {}),
|
||||
...(completed ? { sourceGraphGeneration: vector.graphGeneration } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
return exported;
|
||||
}
|
||||
|
||||
/** Commit metadata intentionally omits checkout-only required RepoMeta fields. */
|
||||
function commitMetadata(meta: RepoMeta): RepoMeta {
|
||||
const result: Record<string, unknown> = { ...meta };
|
||||
for (const field of CHECKOUT_FIELDS) delete result[field];
|
||||
return result as unknown as RepoMeta;
|
||||
}
|
||||
|
||||
const exists = (p: string): Promise<boolean> =>
|
||||
fs.access(p).then(
|
||||
() => true,
|
||||
|
|
@ -204,7 +314,7 @@ const seedFromLocalIndex = async (
|
|||
source: string,
|
||||
log: Log,
|
||||
): Promise<boolean> => {
|
||||
const sourceGraph = path.join(source, LBUG_DIRECTORY);
|
||||
let sourceGraph = resolveGraphPath(source);
|
||||
const usable = (m: RepoMeta | null): m is RepoMeta =>
|
||||
!!m &&
|
||||
!m.incrementalInProgress &&
|
||||
|
|
@ -219,13 +329,22 @@ const seedFromLocalIndex = async (
|
|||
}
|
||||
let meta: RepoMeta;
|
||||
try {
|
||||
if (lock.lockFree || (await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false;
|
||||
if (lock.lockFree) return false;
|
||||
// Re-read under the lock: an analyze that finished while this waited
|
||||
// rewrote both, and the copied graph must match the saved metadata.
|
||||
const locked = await loadMeta(source);
|
||||
if (!usable(locked)) return false;
|
||||
meta = locked;
|
||||
sourceGraph = resolveGraphPath(source);
|
||||
if ((await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false;
|
||||
if (
|
||||
locked.embeddingJob &&
|
||||
registryPathEquals(canonicalizePath(locked.repoPath), canonicalizePath(repoPath))
|
||||
)
|
||||
throw new Error(
|
||||
"Finish this checkout's pending embedding job in its original storage before shared-store adoption",
|
||||
);
|
||||
await fs.mkdir(slot, { recursive: true });
|
||||
meta = await retainSharedMetadata(source, slot, locked, true);
|
||||
try {
|
||||
await cloneGraphFile(sourceGraph, path.join(slot, LBUG_DIRECTORY));
|
||||
} catch (err) {
|
||||
|
|
@ -279,13 +398,18 @@ export const seedSharedSlot = async (
|
|||
(await withStoreLock(layout, 'publish', async () => {
|
||||
const graph = path.join(seed.dir, LBUG_DIRECTORY);
|
||||
if (!(await exists(graph))) return false;
|
||||
// Vector publication can advance this snapshot after pickSeed. Re-read
|
||||
// while the publish lock also prevents its source from being collected.
|
||||
const current = await loadMeta(seed.dir);
|
||||
if (!current) return false;
|
||||
await fs.mkdir(layout.checkoutSlot, { recursive: true });
|
||||
const retained = await retainSharedMetadata(seed.dir, layout.checkoutSlot, current);
|
||||
const meta: RepoMeta = {
|
||||
...seed.meta,
|
||||
...retained,
|
||||
repoPath,
|
||||
storagePath: layout.checkoutSlot,
|
||||
graphPath: graph,
|
||||
lastCommit: upToDate ? seed.meta.lastCommit : '',
|
||||
lastCommit: upToDate && canReuseGraph(current) ? current.lastCommit : '',
|
||||
};
|
||||
delete meta.incrementalInProgress;
|
||||
await saveMeta(layout.checkoutSlot, meta);
|
||||
|
|
@ -316,7 +440,7 @@ export const ensurePrivateSharedGraph = async (
|
|||
log: Log,
|
||||
opts: { copy?: boolean } = {},
|
||||
): Promise<boolean> => {
|
||||
const own = path.join(slot, LBUG_DIRECTORY);
|
||||
const own = resolveGraphWritePath(slot);
|
||||
const pointed = resolveGraphPath(slot);
|
||||
if (pointed === own) return true;
|
||||
const meta = await loadMeta(slot);
|
||||
|
|
@ -384,7 +508,7 @@ export const publishSharedGraph = async (
|
|||
const slot = layout.checkoutSlot;
|
||||
const meta = await loadMeta(slot);
|
||||
if (!meta) return;
|
||||
const own = path.join(slot, LBUG_DIRECTORY);
|
||||
const own = resolveGraphWritePath(slot);
|
||||
|
||||
// A graph built while files were dirty must not become shared until analyze
|
||||
// has reconciled those paths with disk and cleared them from the coverage receipt.
|
||||
|
|
@ -397,10 +521,14 @@ export const publishSharedGraph = async (
|
|||
// FTS-only repair preserves the graph's existing parser provenance.
|
||||
meta.parseCacheIdentity !== undefined &&
|
||||
!meta.incrementalInProgress &&
|
||||
!meta.graphWriteCollapsed &&
|
||||
builtClean &&
|
||||
// Embeddings still owed stay with this checkout, which finishes them; a
|
||||
// commit graph never changes, so a shared copy would stay short for good.
|
||||
!meta.embeddingCheckpoint &&
|
||||
// A migration receipt owns historical vectors in this private slot. Do
|
||||
// not replace its graph or strip that evidence before migration finishes.
|
||||
!Object.hasOwn(meta, 'legacyVectorSource') &&
|
||||
// A sparse or partial checkout builds a graph missing the files it hides.
|
||||
// Every sparse mode marks those entries skip-worktree, which this rejects
|
||||
// (git-utils.test.ts covers no-cone, cone and sparse-index checkouts).
|
||||
|
|
@ -421,8 +549,9 @@ export const publishSharedGraph = async (
|
|||
const ownIsBetter =
|
||||
published &&
|
||||
(!targetMeta ||
|
||||
!!targetMeta.embeddingCheckpoint ||
|
||||
(targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)) &&
|
||||
(!hasSeparateGraphGeneration(meta) &&
|
||||
(!!targetMeta.embeddingCheckpoint ||
|
||||
(targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)))) &&
|
||||
(await exists(own));
|
||||
if (ownIsBetter) {
|
||||
published = false;
|
||||
|
|
@ -431,22 +560,25 @@ export const publishSharedGraph = async (
|
|||
);
|
||||
} else if (published) {
|
||||
try {
|
||||
if (hasSeparateGraphGeneration(meta) && meta.vectorGeneration && !meta.embeddingJob) {
|
||||
const retained = await retainSharedMetadata(slot, target, meta);
|
||||
if (retained.vectorGeneration) await saveMeta(target, commitMetadata(retained));
|
||||
}
|
||||
await wipeLbugDbFiles(own);
|
||||
} catch (err) {
|
||||
// An open reader (Windows) can block the delete. The analysis
|
||||
// already succeeded; keep the private graph and try next run.
|
||||
published = false;
|
||||
log(
|
||||
`Shared store: could not drop the private graph (${(err as Error).message}); keeping it.`,
|
||||
`Shared store: could not finalize the shared graph (${(err as Error).message}); keeping the existing graph.`,
|
||||
);
|
||||
}
|
||||
} else if ((await exists(own)) && (await inspectLbugSidecars(own)).kind === 'clean') {
|
||||
await fs.mkdir(layout.commitsDir, { recursive: true });
|
||||
const staging = path.join(layout.commitsDir, `.publish-${randomUUID()}`);
|
||||
await fs.mkdir(staging);
|
||||
const commitMeta: Record<string, unknown> = { ...meta };
|
||||
for (const field of CHECKOUT_FIELDS) delete commitMeta[field];
|
||||
try {
|
||||
const commitMeta = commitMetadata(await retainSharedMetadata(slot, staging, meta));
|
||||
await fs.rename(own, path.join(staging, LBUG_DIRECTORY));
|
||||
await fs.writeFile(path.join(staging, INDEX_METADATA_FILE), JSON.stringify(commitMeta));
|
||||
await fs.rename(staging, target);
|
||||
|
|
@ -481,7 +613,11 @@ export const publishSharedGraph = async (
|
|||
meta.graphPath = targetGraph;
|
||||
await saveMeta(slot, meta);
|
||||
}
|
||||
} else if (meta.graphPath !== undefined && (await exists(own))) {
|
||||
} else if (
|
||||
meta.graphPath !== undefined &&
|
||||
path.isAbsolute(meta.graphPath) &&
|
||||
(await exists(own))
|
||||
) {
|
||||
delete meta.graphPath;
|
||||
await saveMeta(slot, meta);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,6 +116,8 @@ import {
|
|||
embeddingDimsMismatch,
|
||||
} from '../../core/lbug/schema.js';
|
||||
import { getExactScanLimit } from '../../core/platform/capabilities.js';
|
||||
import { metadataMatchesGraph, selectVectorGeneration } from '../../storage/vector-generation.js';
|
||||
import { pinRepo } from '../../core/lbug/pool-adapter.js';
|
||||
import { PhaseTimer } from '../../core/search/phase-timer.js';
|
||||
import { ftsDegradedWarning, ftsQueryFailedWarning } from '../../core/search/fts-indexes.js';
|
||||
import { getFtsDisabledReason, type FtsDisabledReason } from '../../core/search/fts-policy.js';
|
||||
|
|
@ -1850,6 +1852,17 @@ export class LocalBackend {
|
|||
if (!pending) {
|
||||
pending = loadOptionalMeta(path.dirname(repo.lbugPath)).then((meta) => {
|
||||
if (!meta) return null;
|
||||
if (
|
||||
(meta.graphPath !== undefined || meta.graphGeneration) &&
|
||||
!metadataMatchesGraph(path.dirname(repo.lbugPath), meta, repo.lbugPath)
|
||||
) {
|
||||
// A newer immutable path was published after this request resolved
|
||||
// its handle. Keep the old reader's metadata, never mix generations.
|
||||
const previous = this.lastObservedPoolState.get(repo.lbugPath);
|
||||
if (previous?.metadata) return previous.metadata;
|
||||
repo.generationUnknown = true;
|
||||
return null;
|
||||
}
|
||||
// Readers never use ingestion manifests. Do not keep their per-file
|
||||
// hashes/cache keys or pending embedding IDs in an opened snapshot.
|
||||
return {
|
||||
|
|
@ -3562,6 +3575,7 @@ export class LocalBackend {
|
|||
chain_depth?: number;
|
||||
},
|
||||
): Promise<any> {
|
||||
const release = pinRepo(repo.lbugPath);
|
||||
try {
|
||||
const result = await this._queryImpl(repo, params);
|
||||
if (!result.error) assertResponseIdentities(result);
|
||||
|
|
@ -3571,6 +3585,8 @@ export class LocalBackend {
|
|||
return { error: err.message, recoverySuggestion: SYMBOL_IDENTITY_RECOVERY_SUGGESTION };
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4575,55 +4591,103 @@ export class LocalBackend {
|
|||
// call's width, and the catch below must only clear an entry it did not set.
|
||||
let embeddedDims: number | undefined;
|
||||
try {
|
||||
// Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off)
|
||||
// determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only
|
||||
// the count is read, to decide whether to load the embedding model at all.
|
||||
const tableCheck = await executeQuery(
|
||||
repo.lbugPath,
|
||||
`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`,
|
||||
);
|
||||
if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) {
|
||||
if (repo.generationUnknown) {
|
||||
if (degraded)
|
||||
degraded.reason =
|
||||
'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.';
|
||||
this.lastQueryEmbeddingDims.delete(repo.lbugPath);
|
||||
this.logKeywordOnlyEmbeddings();
|
||||
return [];
|
||||
}
|
||||
|
||||
const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js');
|
||||
const queryVec = await embedQuery(query);
|
||||
const dims = getEmbeddingDims();
|
||||
// #2798: the width this query vector really was produced at — the same
|
||||
// value the CAST below binds against the index's `FLOAT[N]` column, and
|
||||
// therefore the only honest comparand for query()'s drift warning.
|
||||
embeddedDims = dims;
|
||||
this.lastQueryEmbeddingDims.set(repo.lbugPath, dims);
|
||||
const queryVecStr = `[${queryVec.join(',')}]`;
|
||||
const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE);
|
||||
let vectorReady = false;
|
||||
try {
|
||||
vectorReady = await ensureVectorExtension(repo.lbugPath);
|
||||
} catch (err) {
|
||||
rethrowSymbolIdentityError(err);
|
||||
if (!this.warnedVectorLoadFailed) {
|
||||
this.warnedVectorLoadFailed = true;
|
||||
logger.warn(
|
||||
{ err },
|
||||
'GitNexus [query:vector]: vector extension load failed; using exact scan fallback',
|
||||
);
|
||||
}
|
||||
const metadata = await this.readToolMetadata(repo);
|
||||
if (repo.generationUnknown) {
|
||||
if (degraded)
|
||||
degraded.reason =
|
||||
'Semantic search is unavailable because the graph generation is unverified. Run gitnexus analyze; graph and keyword search remain available.';
|
||||
this.lastQueryEmbeddingDims.delete(repo.lbugPath);
|
||||
return [];
|
||||
}
|
||||
const selection = selectVectorGeneration(
|
||||
path.dirname(repo.lbugPath),
|
||||
metadata,
|
||||
repo.lbugPath,
|
||||
getExactScanLimit(),
|
||||
);
|
||||
if (selection.kind === 'unavailable') {
|
||||
if (degraded && selection.reason === 'embedding-pending')
|
||||
degraded.reason =
|
||||
'Semantic search is unavailable while embeddings are pending. Run gitnexus analyze to resume; graph and keyword search remain available.';
|
||||
else if (degraded && selection.reason !== 'embeddings-unavailable')
|
||||
degraded.reason = `Semantic search is unavailable (${selection.reason}). Run gitnexus analyze; graph and keyword search remain available.`;
|
||||
this.lastQueryEmbeddingDims.delete(repo.lbugPath);
|
||||
return [];
|
||||
}
|
||||
|
||||
let bestChunks = new Map<
|
||||
string,
|
||||
{ distance: number; chunkIndex: number; startLine: number; endLine: number }
|
||||
>();
|
||||
// Try the vector lane only after its lazy load succeeds. An unavailable
|
||||
// extension is already reported by ExtensionManager; an index/query
|
||||
// failure below gets its own once-per-backend diagnostic before the exact
|
||||
// scan fallback.
|
||||
if (vectorReady) {
|
||||
if (selection.kind === 'ready') {
|
||||
const { searchVectorGeneration } =
|
||||
await import('../../core/embeddings/vector-generation-search.js');
|
||||
const chunks = await searchVectorGeneration(
|
||||
{ metadataDirectory: path.dirname(repo.lbugPath), metadata, graphPath: repo.lbugPath },
|
||||
async () => {
|
||||
const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js');
|
||||
const queryVec = await embedQuery(query);
|
||||
embeddedDims = getEmbeddingDims();
|
||||
this.lastQueryEmbeddingDims.set(repo.lbugPath, embeddedDims);
|
||||
return queryVec;
|
||||
},
|
||||
limit,
|
||||
getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE),
|
||||
);
|
||||
bestChunks = new Map((chunks ?? []).map(({ nodeId, ...chunk }) => [nodeId, chunk]));
|
||||
if (embeddedDims === undefined) this.lastQueryEmbeddingDims.delete(repo.lbugPath);
|
||||
} else {
|
||||
// Check if embedding table exists before loading the model (avoids heavy model init when embeddings are off)
|
||||
// determinism: probe — aggregate singleton. COUNT(*) with no grouping key returns exactly one row, and only
|
||||
// the count is read, to decide whether to load the embedding model at all.
|
||||
const tableCheck = await executeQuery(
|
||||
repo.lbugPath,
|
||||
`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN COUNT(*) AS cnt LIMIT 1`,
|
||||
);
|
||||
if (!tableCheck.length || (tableCheck[0].cnt ?? tableCheck[0][0]) === 0) {
|
||||
this.lastQueryEmbeddingDims.delete(repo.lbugPath);
|
||||
this.logKeywordOnlyEmbeddings();
|
||||
return [];
|
||||
}
|
||||
|
||||
const { embedQuery, getEmbeddingDims } = await import('../core/embedder.js');
|
||||
const queryVec = await embedQuery(query);
|
||||
const dims = getEmbeddingDims();
|
||||
// #2798: the width this query vector really was produced at — the same
|
||||
// value the CAST below binds against the index's `FLOAT[N]` column, and
|
||||
// therefore the only honest comparand for query()'s drift warning.
|
||||
embeddedDims = dims;
|
||||
this.lastQueryEmbeddingDims.set(repo.lbugPath, dims);
|
||||
const queryVecStr = `[${queryVec.join(',')}]`;
|
||||
const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE);
|
||||
let vectorReady = false;
|
||||
try {
|
||||
bestChunks = await collectBestChunks(limit, async (fetchLimit) => {
|
||||
const vectorQuery = `
|
||||
vectorReady = await ensureVectorExtension(repo.lbugPath);
|
||||
} catch (err) {
|
||||
rethrowSymbolIdentityError(err);
|
||||
if (!this.warnedVectorLoadFailed) {
|
||||
this.warnedVectorLoadFailed = true;
|
||||
logger.warn(
|
||||
{ err },
|
||||
'GitNexus [query:vector]: vector extension load failed; using exact scan fallback',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Try the vector lane only after its lazy load succeeds. An unavailable
|
||||
// extension is already reported by ExtensionManager; an index/query
|
||||
// failure below gets its own once-per-backend diagnostic before the exact
|
||||
// scan fallback.
|
||||
if (vectorReady) {
|
||||
try {
|
||||
bestChunks = await collectBestChunks(limit, async (fetchLimit) => {
|
||||
const vectorQuery = `
|
||||
CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
|
||||
CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit})
|
||||
YIELD node AS emb, distance
|
||||
|
|
@ -4634,66 +4698,66 @@ export class LocalBackend {
|
|||
ORDER BY distance
|
||||
`;
|
||||
|
||||
const embResults = await executeQuery(repo.lbugPath, vectorQuery);
|
||||
for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0);
|
||||
return embResults.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
distance: row.distance ?? row[4],
|
||||
}));
|
||||
});
|
||||
} catch (err) {
|
||||
rethrowSymbolIdentityError(err);
|
||||
bestChunks = new Map();
|
||||
if (!this.warnedVectorQueryFailed) {
|
||||
// Rare diagnostic: surface why semantic search fell back to the
|
||||
// exact scan. Emitted once per `LocalBackend` instance lifetime to
|
||||
// avoid noisy stderr on hot semantic-search paths (DoD §2.8).
|
||||
this.warnedVectorQueryFailed = true;
|
||||
logger.warn(
|
||||
{ err },
|
||||
'GitNexus [query:vector]: vector index query failed; using exact scan fallback',
|
||||
);
|
||||
const embResults = await executeQuery(repo.lbugPath, vectorQuery);
|
||||
for (const row of embResults) assertQueryIdentity(row, 'nodeId', 0);
|
||||
return embResults.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
distance: row.distance ?? row[4],
|
||||
}));
|
||||
});
|
||||
} catch (err) {
|
||||
rethrowSymbolIdentityError(err);
|
||||
bestChunks = new Map();
|
||||
if (!this.warnedVectorQueryFailed) {
|
||||
// Rare diagnostic: surface why semantic search fell back to the
|
||||
// exact scan. Emitted once per `LocalBackend` instance lifetime to
|
||||
// avoid noisy stderr on hot semantic-search paths (DoD §2.8).
|
||||
this.warnedVectorQueryFailed = true;
|
||||
logger.warn(
|
||||
{ err },
|
||||
'GitNexus [query:vector]: vector index query failed; using exact scan fallback',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestChunks.size === 0) {
|
||||
const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0);
|
||||
const exactLimit = getExactScanLimit();
|
||||
if (embeddingCount > exactLimit) return [];
|
||||
if (bestChunks.size === 0) {
|
||||
const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0);
|
||||
const exactLimit = getExactScanLimit();
|
||||
if (embeddingCount > exactLimit) return [];
|
||||
|
||||
const rows = await executeQuery(
|
||||
repo.lbugPath,
|
||||
`
|
||||
const rows = await executeQuery(
|
||||
repo.lbugPath,
|
||||
`
|
||||
MATCH (e:${EMBEDDING_TABLE_NAME})
|
||||
RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex,
|
||||
e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding
|
||||
`,
|
||||
);
|
||||
for (const row of rows) assertQueryIdentity(row, 'nodeId', 0);
|
||||
const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
embedding: row.embedding ?? row[4] ?? [],
|
||||
}));
|
||||
bestChunks = new Map(
|
||||
rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [
|
||||
row.nodeId,
|
||||
{
|
||||
distance: row.distance,
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
},
|
||||
]),
|
||||
);
|
||||
);
|
||||
for (const row of rows) assertQueryIdentity(row, 'nodeId', 0);
|
||||
const exactRows: ExactEmbeddingRow[] = rows.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
embedding: row.embedding ?? row[4] ?? [],
|
||||
}));
|
||||
bestChunks = new Map(
|
||||
rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [
|
||||
row.nodeId,
|
||||
{
|
||||
distance: row.distance,
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (bestChunks.size === 0) return [];
|
||||
|
||||
const results: any[] = [];
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ import type { LocalBackend } from './local/local-backend.js';
|
|||
import { checkStaleness } from './staleness.js';
|
||||
import { loadMeta } from '../storage/repo-manager.js';
|
||||
import { ANALYZER_RUNNER_IDENTITY_SCHEMA_VERSION } from '../core/analyzer-identity.js';
|
||||
import { getIndexIncompleteReasons } from '../core/index-freshness.js';
|
||||
import { getIndexIncompleteReasons, getGraphIncompleteReasons } from '../core/index-freshness.js';
|
||||
import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js';
|
||||
import { readIndexCapabilities } from '../storage/index-capabilities.js';
|
||||
import {
|
||||
checkoutIsDirectory,
|
||||
contentRetentionFromMeta,
|
||||
|
|
@ -381,6 +383,11 @@ async function getContextResource(backend: LocalBackend, repoName?: string): Pro
|
|||
lines.push(` source_available: ${JSON.stringify(sourceAvailable)}`);
|
||||
lines.push(` runner_identity: ${JSON.stringify(freshMeta?.runnerIdentity ?? null)}`);
|
||||
lines.push(` incomplete_reasons: ${JSON.stringify(incompleteReasons)}`);
|
||||
lines.push(` graph_incomplete_reasons: ${JSON.stringify(getGraphIncompleteReasons(freshMeta))}`);
|
||||
const embeddingProgress = await readEmbeddingJobProgress(repo.storagePath, freshMeta);
|
||||
if (embeddingProgress) lines.push(` embedding_progress: ${JSON.stringify(embeddingProgress)}`);
|
||||
const capabilities = await readIndexCapabilities(repo.storagePath, freshMeta);
|
||||
if (capabilities) lines.push(` capabilities: ${JSON.stringify(capabilities)}`);
|
||||
lines.push(` spring_actuator: ${JSON.stringify(freshMeta?.springActuator ?? null)}`);
|
||||
// Surfaced beside the Actuator flag, and it matters more than that one does:
|
||||
// Actuator only annotates nodes the source pass already found, whereas
|
||||
|
|
|
|||
|
|
@ -12,8 +12,9 @@
|
|||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import type { ChildProcess } from 'child_process';
|
||||
import type { AnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
|
||||
export interface AnalyzeJobProgress {
|
||||
export interface AnalyzeJobProgress extends AnalysisPhaseProgress {
|
||||
phase: string;
|
||||
percent: number;
|
||||
message: string;
|
||||
|
|
@ -88,8 +89,8 @@ export interface AnalyzeJob {
|
|||
const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
|
||||
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const JOB_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
|
||||
/** How long a cancelled worker gets to exit via IPC before a signal is sent. */
|
||||
const CANCEL_GRACE_MS = 15_000;
|
||||
/** Delay before force-killing an unresponsive standalone child after cancellation. */
|
||||
const CANCEL_GRACE_MS = 60_000;
|
||||
|
||||
export class JobManager {
|
||||
private jobs = new Map<string, AnalyzeJob>();
|
||||
|
|
@ -208,20 +209,27 @@ export class JobManager {
|
|||
|
||||
/** Register a child process for a job — enables cancellation and timeout. */
|
||||
registerChild(jobId: string, child: ChildProcess) {
|
||||
const previousTimer = this.timeouts.get(jobId);
|
||||
if (previousTimer) clearTimeout(previousTimer);
|
||||
this.children.set(jobId, child);
|
||||
|
||||
// 30-minute timeout
|
||||
const timer = setTimeout(() => {
|
||||
const job = this.jobs.get(jobId);
|
||||
if (job && !this.isTerminal(job.status)) {
|
||||
this.cancelJob(jobId, 'Analysis timed out (30 minute limit)');
|
||||
}
|
||||
}, JOB_TIMEOUT_MS);
|
||||
// One deadline covers graph preparation, continuation, and crash retries.
|
||||
const elapsed = Date.now() - (this.jobs.get(jobId)?.startedAt ?? Date.now());
|
||||
const timer = setTimeout(
|
||||
() => {
|
||||
const job = this.jobs.get(jobId);
|
||||
if (job && !this.isTerminal(job.status)) {
|
||||
this.cancelJob(jobId, 'Analysis timed out (30 minute limit)');
|
||||
}
|
||||
},
|
||||
Math.max(0, JOB_TIMEOUT_MS - elapsed),
|
||||
);
|
||||
this.timeouts.set(jobId, timer);
|
||||
|
||||
// Apply a pending cancel BEFORE other `exit` listeners (analyze-launch's
|
||||
// crash-retry) see a still-non-terminal job and fork a replacement worker.
|
||||
const onExit = (): void => {
|
||||
if (this.children.get(jobId) !== child) return;
|
||||
this.releaseChild(jobId);
|
||||
this.applyPendingCancel(jobId);
|
||||
};
|
||||
|
|
@ -286,12 +294,8 @@ export class JobManager {
|
|||
/**
|
||||
* Cancel a running job.
|
||||
*
|
||||
* The worker is asked to stop over IPC first (`{ type: 'cancel' }`), which
|
||||
* lets it reach a JS-visible safe point and checkpoint before exiting —
|
||||
* the same cross-platform control path `core/auto-sync` uses. A signal is
|
||||
* sent only as a bounded fallback: on Windows `child.kill('SIGTERM')` is a
|
||||
* forceful termination (Node ignores the signal name there), so leading
|
||||
* with it could kill the worker mid LadybugDB write.
|
||||
* Request cooperative cancellation over IPC first. A standalone child that
|
||||
* outlasts the grace period is killed; ownership remains held until its exit.
|
||||
*/
|
||||
cancelJob(jobId: string, reason?: string): boolean {
|
||||
const job = this.jobs.get(jobId);
|
||||
|
|
@ -328,31 +332,25 @@ export class JobManager {
|
|||
}
|
||||
|
||||
/**
|
||||
* Ask a worker to shut down: IPC cancel now, signal after a grace period if
|
||||
* it has not exited on its own. The grace timer is cleared by the child's
|
||||
* `exit` handler registered in `registerChild`.
|
||||
* Request cancellation, then stop a wedged standalone process after grace.
|
||||
* Ownership and the job slot remain held until the actual process exit.
|
||||
*/
|
||||
private requestChildShutdown(jobId: string, child: ChildProcess): void {
|
||||
let ipcSent = false;
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'cancel' });
|
||||
ipcSent = true;
|
||||
child.send({ type: 'cancel' }, () => {
|
||||
/* Disconnect also requests cancellation. */
|
||||
});
|
||||
} catch {
|
||||
// Channel already closed — fall through to the signal path.
|
||||
// The child's disconnect handler cancels without interrupting native work.
|
||||
}
|
||||
}
|
||||
if (!ipcSent) {
|
||||
child.kill('SIGTERM');
|
||||
return;
|
||||
}
|
||||
if (this.cancelGraceTimers.has(jobId)) return;
|
||||
const grace = setTimeout(() => {
|
||||
this.cancelGraceTimers.delete(jobId);
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
// IPC already set the worker's cooperative cancel flag; a second
|
||||
// SIGTERM is a no-op there. SIGKILL is the actual bounded fallback
|
||||
// (on Windows `kill()` is already TerminateProcess).
|
||||
// Process termination preserves crash-recovery receipts. Never use
|
||||
// this policy for worker_threads running native code in this process.
|
||||
child.kill('SIGKILL');
|
||||
}
|
||||
}, CANCEL_GRACE_MS);
|
||||
|
|
@ -361,23 +359,14 @@ export class JobManager {
|
|||
}
|
||||
|
||||
dispose() {
|
||||
// IPC first so a worker that checks in can stop at a safe point.
|
||||
// On Windows `child.kill('SIGTERM')` is TerminateProcess — skip that
|
||||
// immediate kill and leave the 15s grace timer to SIGKILL. On Unix,
|
||||
// SIGTERM after IPC is cooperative, so the grace timers can be dropped.
|
||||
const windows = process.platform === 'win32';
|
||||
// Cancel jobs between workers too, so an already scheduled retry cannot fork.
|
||||
for (const job of this.jobs.values()) {
|
||||
if (!this.hasPendingCancel(job.id)) this.cancelJob(job.id, 'Server shutting down');
|
||||
}
|
||||
// Cooperative cancellation precedes bounded standalone-process shutdown.
|
||||
for (const [jobId, child] of this.children) {
|
||||
this.requestChildShutdown(jobId, child);
|
||||
if (!windows) {
|
||||
child.kill('SIGTERM');
|
||||
}
|
||||
if (!this.hasPendingCancel(jobId)) this.requestChildShutdown(jobId, child);
|
||||
}
|
||||
this.children.clear();
|
||||
if (!windows) {
|
||||
for (const timer of this.cancelGraceTimers.values()) clearTimeout(timer);
|
||||
this.cancelGraceTimers.clear();
|
||||
}
|
||||
this.pendingCancelReasons.clear();
|
||||
for (const controller of this.abortControllers.values()) controller.abort();
|
||||
this.abortControllers.clear();
|
||||
|
||||
|
|
@ -391,6 +380,29 @@ export class JobManager {
|
|||
this.emitter.removeAllListeners();
|
||||
}
|
||||
|
||||
/** Keep shutdown alive until standalone writers have physically exited. */
|
||||
async waitForChildren(): Promise<void> {
|
||||
await Promise.all(
|
||||
[...this.children.values()].map(
|
||||
(child) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const finished = () => {
|
||||
child.removeListener('exit', finished);
|
||||
child.removeListener('close', finished);
|
||||
resolve();
|
||||
};
|
||||
child.once('exit', finished);
|
||||
// Spawn failures emit error and close, but never exit.
|
||||
child.once('close', finished);
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private isTerminal(status: AnalyzeJob['status']): boolean {
|
||||
return isTerminalJobStatus(status);
|
||||
}
|
||||
|
|
@ -402,7 +414,12 @@ export class JobManager {
|
|||
private cleanup() {
|
||||
const now = Date.now();
|
||||
for (const [id, job] of this.jobs) {
|
||||
if (this.isTerminal(job.status) && job.completedAt && now - job.completedAt > JOB_TTL_MS) {
|
||||
if (
|
||||
!this.children.has(id) &&
|
||||
this.isTerminal(job.status) &&
|
||||
job.completedAt &&
|
||||
now - job.completedAt > JOB_TTL_MS
|
||||
) {
|
||||
this.jobs.delete(id);
|
||||
this.abortControllers.delete(id);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import { fork } from 'child_process';
|
|||
import { fileURLToPath, pathToFileURL } from 'url';
|
||||
import { createRequire } from 'node:module';
|
||||
import { INDEX_METADATA_FILE } from '../storage/repo-manager.js';
|
||||
import { LBUG_DIRECTORY } from '../storage/storage-constants.js';
|
||||
import {
|
||||
ANALYZE_FORCE_STORAGE_REQUIREMENTS,
|
||||
ANALYZE_STORAGE_REQUIREMENTS,
|
||||
|
|
@ -25,11 +24,19 @@ import {
|
|||
} from '../storage/storage-resolver.js';
|
||||
import { BRANCHES_DIR, branchSlug } from '../storage/branch-index.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
import {
|
||||
analysisMayEmbed,
|
||||
analysisSupervisorReservation,
|
||||
resolveHostAnalysisResources,
|
||||
} from '../core/analysis-host-resources.js';
|
||||
import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js';
|
||||
import type { ContinueMessage, GraphPreparedMessage } from './analyze-worker-protocol.js';
|
||||
import { loadAnalyzeConfig } from '../cli/analyze-config.js';
|
||||
import type { RouteHookConfig } from '../core/ingestion/language-provider.js';
|
||||
import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js';
|
||||
import { isTerminalJobStatus, type JobManager } from './analyze-job.js';
|
||||
import type { WorkerMessage } from './analyze-worker.js';
|
||||
import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js';
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
|
||||
|
|
@ -164,7 +171,7 @@ const waitForSettledIndex = async (
|
|||
}
|
||||
}
|
||||
|
||||
const lbugPath = path.resolve(probeRoot, LBUG_DIRECTORY);
|
||||
const lbugPath = path.resolve(resolveGraphPath(probeRoot));
|
||||
const lbugRel = path.relative(storageRoot, lbugPath);
|
||||
if (lbugRel.startsWith('..') || path.isAbsolute(lbugRel)) {
|
||||
return false;
|
||||
|
|
@ -172,8 +179,8 @@ const waitForSettledIndex = async (
|
|||
const lbugStat = statSync(lbugPath);
|
||||
if (lbugStat.mtimeMs < jobStartMs) return false;
|
||||
|
||||
return ['lbug.wal', 'lbug.shadow', 'lbug.wal.checkpoint'].every((name) => {
|
||||
const sidePath = path.resolve(probeRoot, name);
|
||||
return ['.wal', '.shadow', '.wal.checkpoint'].every((suffix) => {
|
||||
const sidePath = path.resolve(`${lbugPath}${suffix}`);
|
||||
const sideRel = path.relative(storageRoot, sidePath);
|
||||
if (sideRel.startsWith('..') || path.isAbsolute(sideRel)) {
|
||||
return false;
|
||||
|
|
@ -223,6 +230,15 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
// For waitForSettledIndex: files (re)written by this job have mtimes at or
|
||||
// after this instant. Taken before the fork so no worker write predates it.
|
||||
const jobStartMs = Date.now();
|
||||
const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB);
|
||||
const resources = resolveHostAnalysisResources({
|
||||
mainHeapMb:
|
||||
(Number.isInteger(envHeapMb) && envHeapMb > 0
|
||||
? envHeapMb
|
||||
: Math.min(8192, autoHeapCapMb())) + 48,
|
||||
includeEmbeddings: await analysisMayEmbed(targetPath, opts),
|
||||
supervisorBytes: analysisSupervisorReservation(),
|
||||
});
|
||||
const analyzeLockKey = await requireStoragePath(
|
||||
targetPath,
|
||||
opts.force ? ANALYZE_FORCE_STORAGE_REQUIREMENTS : ANALYZE_STORAGE_REQUIREMENTS,
|
||||
|
|
@ -270,6 +286,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
'analyze launch: forwarding route configuration to the worker',
|
||||
);
|
||||
|
||||
const analysisOptions = {
|
||||
...opts,
|
||||
graphPublication: 'auto' as const,
|
||||
resources,
|
||||
routeHookConfig,
|
||||
};
|
||||
|
||||
jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
|
||||
|
||||
// ── Worker fork with auto-retry ──────────────────────────────
|
||||
|
|
@ -281,20 +304,12 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
|
||||
: [];
|
||||
|
||||
// Worker heap: 8192MB historical default, but never above what this
|
||||
// machine/container actually has (#2649 review — a fixed 8192 inside a
|
||||
// smaller cgroup limit died to the kernel with a misleading remedy).
|
||||
// GITNEXUS_SERVER_ANALYZE_HEAP_MB overrides as an absolute value.
|
||||
const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB);
|
||||
const workerHeapMb =
|
||||
Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb : Math.min(8192, autoHeapCapMb());
|
||||
|
||||
const launchAborted = (jobId: string): boolean => {
|
||||
const current = jobManager.getJob(jobId);
|
||||
return !current || isTerminalJobStatus(current.status) || jobManager.hasPendingCancel(jobId);
|
||||
};
|
||||
|
||||
const forkWorker = () => {
|
||||
const forkWorker = (continuation?: ContinueMessage) => {
|
||||
if (launchAborted(job.id)) {
|
||||
// Cancelled (or timed out) between lock acquisition and the fork, or
|
||||
// during a crash-retry delay. A pending-cancel job stays non-terminal
|
||||
|
|
@ -305,7 +320,11 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
}
|
||||
|
||||
const child = fork(workerPath, [], {
|
||||
execArgv: [...tsxHookArgs, `--max-old-space-size=${workerHeapMb}`],
|
||||
execArgv: [
|
||||
...tsxHookArgs,
|
||||
`--max-old-space-size=${resources.mainHeapMb - 48}`,
|
||||
'--max-semi-space-size=16',
|
||||
],
|
||||
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
||||
});
|
||||
|
||||
|
|
@ -325,12 +344,15 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
// "don't release on cancel error" would leak the lock.
|
||||
let holdLockUntilExit = false;
|
||||
let childExited = false;
|
||||
let graphVisible = false;
|
||||
let prepared: GraphPreparedMessage | undefined;
|
||||
let terminalMessage: Extract<WorkerMessage, { type: 'complete' | 'error' }> | undefined;
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
stderrChunks += chunk.toString();
|
||||
if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096);
|
||||
});
|
||||
|
||||
child.on('message', (msg: WorkerMessage) => {
|
||||
const handleMessage = (msg: WorkerMessage) => {
|
||||
// Ignore any message once the job is terminal — a late worker message (a
|
||||
// SIGTERM-driven `error` after `complete`, or vice versa) must not
|
||||
// re-release the repo lock or flip the reported status. Mirrors the `exit`
|
||||
|
|
@ -338,12 +360,64 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
const current = jobManager.getJob(job.id);
|
||||
if (!current || isTerminalJobStatus(current.status)) return;
|
||||
|
||||
if ((msg.type === 'complete' || msg.type === 'error') && !childExited) {
|
||||
terminalIpcSeen = true;
|
||||
terminalMessage ??= msg;
|
||||
if (jobManager.applyPendingCancel(job.id)) holdLockUntilExit = true;
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'graph-prepared') {
|
||||
try {
|
||||
if (prepared || continuation) throw new Error('Repeated graph preparation');
|
||||
prepared = { ...msg, handoff: projectPreparedAnalysisHandoff(msg.handoff) };
|
||||
} catch (error) {
|
||||
terminalMessage = { type: 'error', message: (error as Error).message };
|
||||
terminalIpcSeen = true;
|
||||
child.send({ type: 'cancel' });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'graph-ready') {
|
||||
// Core has closed the writer and verified publication. Refresh readers
|
||||
// while this job is still active, then admit the independent vector phase.
|
||||
void closeDbHandle()
|
||||
.then(() => backend.init())
|
||||
.then(() => {
|
||||
if (childExited) return;
|
||||
if (launchAborted(job.id)) child.send({ type: 'cancel' });
|
||||
else {
|
||||
graphVisible = true;
|
||||
child.send({ type: 'graph-visible' });
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (childExited) return;
|
||||
terminalMessage = {
|
||||
type: 'error',
|
||||
message: `Server failed to reload the published graph: ${(error as Error).message}`,
|
||||
};
|
||||
terminalIpcSeen = true;
|
||||
try {
|
||||
child.send({ type: 'cancel' });
|
||||
} catch {
|
||||
/* Wait for actual exit. */
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'complete' || msg.type === 'error') terminalIpcSeen = true;
|
||||
|
||||
if (msg.type === 'progress') {
|
||||
jobManager.updateJob(job.id, {
|
||||
status: 'analyzing',
|
||||
progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
|
||||
progress: {
|
||||
...projectAnalysisPhaseProgress(current.progress),
|
||||
...projectAnalysisPhaseProgress(msg),
|
||||
phase: msg.phase,
|
||||
percent: msg.percent,
|
||||
message: msg.message,
|
||||
},
|
||||
});
|
||||
} else if (msg.type === 'complete') {
|
||||
if (jobManager.applyPendingCancel(job.id)) {
|
||||
|
|
@ -374,20 +448,21 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
// would spin the full 60s and hold the single global analyze slot.
|
||||
// ftsRepairedOnly DOES rewrite `lbug` (initLbug + createSearchFTSIndexes)
|
||||
// so it still waits.
|
||||
const settle = msg.result.alreadyUpToDate
|
||||
? Promise.resolve(true)
|
||||
: waitForSettledIndex(
|
||||
// A checkout's first analyze may write a shared-store slot the
|
||||
// launcher's pre-run lookup could not see yet (#3352); the
|
||||
// worker reports it.
|
||||
(msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath)
|
||||
? msg.result.storagePath
|
||||
: null) ?? analyzeLockKey,
|
||||
jobStartMs,
|
||||
opts.branch,
|
||||
msg.result.isPrimaryBranch,
|
||||
() => launchAborted(job.id),
|
||||
);
|
||||
const settle =
|
||||
continuation || graphVisible || msg.result.alreadyUpToDate
|
||||
? Promise.resolve(true)
|
||||
: waitForSettledIndex(
|
||||
// A checkout's first analyze may write a shared-store slot the
|
||||
// launcher's pre-run lookup could not see yet (#3352); the
|
||||
// worker reports it.
|
||||
(msg.result.storagePath && storeRootOfCheckoutSlot(msg.result.storagePath)
|
||||
? msg.result.storagePath
|
||||
: null) ?? analyzeLockKey,
|
||||
jobStartMs,
|
||||
opts.branch,
|
||||
msg.result.isPrimaryBranch,
|
||||
() => launchAborted(job.id),
|
||||
);
|
||||
settle
|
||||
.then((settled) => {
|
||||
if (launchAborted(job.id)) {
|
||||
|
|
@ -508,7 +583,8 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
jobManager.updateJob(job.id, { status: 'failed', error: msg.message });
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
child.on('message', handleMessage);
|
||||
|
||||
child.on('error', (err) => {
|
||||
// Fake test children have no `pid`. Treat that as pre-spawn so the
|
||||
|
|
@ -532,6 +608,18 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
|
||||
child.on('exit', (code) => {
|
||||
childExited = true;
|
||||
if (terminalMessage) {
|
||||
handleMessage(
|
||||
code === 0
|
||||
? terminalMessage
|
||||
: {
|
||||
type: 'error',
|
||||
message: `Analyze worker exited unsuccessfully (${code ?? 'signal'})`,
|
||||
},
|
||||
);
|
||||
if (launchAborted(job.id)) releaseLockOnce();
|
||||
return;
|
||||
}
|
||||
const j = jobManager.getJob(job.id);
|
||||
if (!j || isTerminalJobStatus(j.status) || jobManager.hasPendingCancel(job.id)) {
|
||||
// (a) complete/error IPC is in flight (`terminalIpcSeen`) — that
|
||||
|
|
@ -553,6 +641,17 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
// finished, successful analysis.
|
||||
if (terminalIpcSeen) return;
|
||||
|
||||
if (prepared && code === 0 && child.pid) {
|
||||
forkWorker({
|
||||
type: 'continue',
|
||||
repoPath: targetPath,
|
||||
options: analysisOptions,
|
||||
writerPid: child.pid,
|
||||
prepared,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Worker crashed — attempt retry if under the limit
|
||||
if (j.retryCount < MAX_WORKER_RETRIES) {
|
||||
j.retryCount++;
|
||||
|
|
@ -571,7 +670,7 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
},
|
||||
});
|
||||
stderrChunks = '';
|
||||
setTimeout(forkWorker, delay);
|
||||
setTimeout(() => forkWorker(), delay);
|
||||
} else {
|
||||
// Exhausted retries — permanent failure
|
||||
releaseLockOnce();
|
||||
|
|
@ -586,20 +685,13 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) {
|
|||
jobManager.registerChild(job.id, child);
|
||||
|
||||
// Send start command to child
|
||||
child.send({
|
||||
type: 'start',
|
||||
repoPath: targetPath,
|
||||
options: {
|
||||
force: !!opts.force,
|
||||
embeddings: !!opts.embeddings,
|
||||
dropEmbeddings: !!opts.dropEmbeddings,
|
||||
...(opts.springActuatorPath ? { springActuatorPath: opts.springActuatorPath } : {}),
|
||||
...(opts.asyncApiSpecPath ? { asyncApiSpecPath: opts.asyncApiSpecPath } : {}),
|
||||
...(opts.registryName ? { registryName: opts.registryName } : {}),
|
||||
...(opts.branch ? { branch: opts.branch } : {}),
|
||||
routeHookConfig,
|
||||
child.send(
|
||||
continuation ?? {
|
||||
type: 'start',
|
||||
repoPath: targetPath,
|
||||
options: analysisOptions,
|
||||
},
|
||||
});
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -13,19 +13,24 @@
|
|||
* worker — only the lightweight `analyze-worker-ipc` projection helper.
|
||||
*/
|
||||
import type { AnalyzeOptions } from '../core/run-analyze.js';
|
||||
import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
// The IPC message protocol lives in a declarations-only leaf, NOT in the
|
||||
// `analyze-worker.ts` entry module: importing it from the entry made this
|
||||
// module depend on the very module that depends on it (#cycle). Erased at
|
||||
// runtime either way; this way the graph is acyclic too.
|
||||
import type { WorkerMessage } from './analyze-worker-protocol.js';
|
||||
import type { WorkerMessage, ContinueMessage } from './analyze-worker-protocol.js';
|
||||
import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js';
|
||||
import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js';
|
||||
import { projectPreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js';
|
||||
// Value import (instanceof): index-lock is a lightweight storage primitive
|
||||
// (node:fs/net/crypto only), so this does NOT pull in run-analyze/repo-manager.
|
||||
import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index-lock.js';
|
||||
|
||||
export interface WorkerAnalysisDeps {
|
||||
runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis;
|
||||
continuePreparedAnalysis?: typeof import('../core/run-analyze.js').continuePreparedAnalysis;
|
||||
onGraphPublished?: () => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
assertAnalysisFinalized: typeof import('../storage/repo-manager.js').assertAnalysisFinalized;
|
||||
send: (msg: WorkerMessage) => void;
|
||||
/**
|
||||
|
|
@ -35,6 +40,9 @@ export interface WorkerAnalysisDeps {
|
|||
* both report a terminal outcome (#2264 P3). See {@link createTerminalClaim}.
|
||||
*/
|
||||
claimTerminal: () => boolean;
|
||||
/** Required for nonterminal graph handoff; never claim the final outcome just
|
||||
* to test whether cancellation already owns it. */
|
||||
isCancelled?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -48,25 +56,66 @@ export async function runWorkerAnalysis(
|
|||
options: AnalyzeOptions,
|
||||
deps: WorkerAnalysisDeps,
|
||||
runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity,
|
||||
continuation?: ContinueMessage,
|
||||
): Promise<void> {
|
||||
let terminal: WorkerMessage;
|
||||
try {
|
||||
const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap
|
||||
? [runnerIdentityAtBootstrap]
|
||||
: [];
|
||||
const result = await deps.runFullAnalysis(
|
||||
repoPath,
|
||||
// This worker force-exits right after reporting, so skip the native close
|
||||
// (it can double-free in LadybugDB's ClientContext destructor after --pdg
|
||||
// writes); flushWAL still persists the index, process.exit reclaims handles.
|
||||
{ ...options, skipNativeCloseOnExit: true },
|
||||
{
|
||||
onProgress: (phase, percent, message) =>
|
||||
deps.send({ type: 'progress', phase, percent, message }),
|
||||
onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }),
|
||||
},
|
||||
...bootstrapArgs,
|
||||
);
|
||||
const callbacks: import('../core/run-analyze.js').AnalyzeCallbacks = {
|
||||
signal: deps.signal,
|
||||
onGraphPublished: deps.onGraphPublished,
|
||||
onProgress: (phase, percent, message, details) =>
|
||||
deps.send({
|
||||
type: 'progress',
|
||||
phase,
|
||||
percent,
|
||||
message,
|
||||
...projectAnalysisPhaseProgress(details),
|
||||
}),
|
||||
onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }),
|
||||
};
|
||||
let result: import('../core/run-analyze.js').AnalyzeResult;
|
||||
if (continuation) {
|
||||
if (!deps.continuePreparedAnalysis) throw new Error('Missing analysis continuation');
|
||||
const handoff = projectPreparedAnalysisHandoff(continuation.prepared.handoff);
|
||||
if (handoff.repoPath !== repoPath || continuation.prepared.result.repoPath !== repoPath)
|
||||
throw new Error('Continuation belongs to a different repository');
|
||||
const metadata = await deps.continuePreparedAnalysis(
|
||||
handoff,
|
||||
continuation.writerPid,
|
||||
options,
|
||||
callbacks,
|
||||
deps.signal,
|
||||
);
|
||||
result = {
|
||||
...continuation.prepared.result,
|
||||
stats: metadata.stats ?? {},
|
||||
rebuildReasons: [],
|
||||
};
|
||||
} else {
|
||||
// Native allocations belong to this exit-only graph process.
|
||||
result = await deps.runFullAnalysis(
|
||||
repoPath,
|
||||
{ ...options, skipNativeCloseOnExit: true },
|
||||
callbacks,
|
||||
...bootstrapArgs,
|
||||
);
|
||||
}
|
||||
if (result.preparedGraph) {
|
||||
if (!deps.isCancelled) throw new Error('Graph handoff requires cancellation state');
|
||||
if (result.graphWriteCollapsed)
|
||||
throw new Error('Collapsed graph cannot be prepared for publication');
|
||||
if (!deps.isCancelled()) {
|
||||
deps.send({
|
||||
type: 'graph-prepared',
|
||||
handoff: projectPreparedAnalysisHandoff(result.preparedGraph),
|
||||
result: projectAnalyzeResultForIpc(result),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
// P2 (#2264): a half-finalized repo — meta.json written but the global
|
||||
// registry entry missing (e.g. a prior collision-aborted run, or a wiped
|
||||
// registry) — must NOT be reported as a successful analysis. Mirror the CLI's
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@
|
|||
* sites (`analyze-launch.ts`, tests) keep working unchanged.
|
||||
*/
|
||||
import type { AnalyzeOptions } from '../core/run-analyze.js';
|
||||
import type { AnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
import type { AnalyzeResultIpc } from './analyze-worker-ipc.js';
|
||||
import type { PreparedAnalysisHandoff } from '../core/analysis-phase-supervisor.js';
|
||||
|
||||
/** Parent → child: start one analysis run. */
|
||||
export interface StartMessage {
|
||||
|
|
@ -33,14 +35,27 @@ export interface StartMessage {
|
|||
options: AnalyzeOptions;
|
||||
}
|
||||
|
||||
/** A new process may continue only after the graph writer actually exited. */
|
||||
export interface ContinueMessage {
|
||||
type: 'continue';
|
||||
repoPath: string;
|
||||
options: AnalyzeOptions;
|
||||
writerPid: number;
|
||||
prepared: GraphPreparedMessage;
|
||||
}
|
||||
|
||||
export interface GraphVisibleMessage {
|
||||
type: 'graph-visible';
|
||||
}
|
||||
|
||||
/** Parent → child: request safe cancellation at the next JS-visible checkpoint. */
|
||||
export interface CancelMessage {
|
||||
type: 'cancel';
|
||||
}
|
||||
|
||||
export type ParentMessage = StartMessage | CancelMessage;
|
||||
export type ParentMessage = StartMessage | ContinueMessage | CancelMessage | GraphVisibleMessage;
|
||||
|
||||
export interface ProgressMessage {
|
||||
export interface ProgressMessage extends AnalysisPhaseProgress {
|
||||
type: 'progress';
|
||||
phase: string;
|
||||
percent: number;
|
||||
|
|
@ -54,6 +69,18 @@ export interface CompleteMessage {
|
|||
result: AnalyzeResultIpc;
|
||||
}
|
||||
|
||||
/** Nonterminal: the graph process still owns its native allocations. */
|
||||
export interface GraphPreparedMessage {
|
||||
type: 'graph-prepared';
|
||||
handoff: PreparedAnalysisHandoff;
|
||||
result: AnalyzeResultIpc;
|
||||
}
|
||||
|
||||
/** Wait for the serving parent to refresh readers before making paid requests. */
|
||||
export interface GraphReadyMessage {
|
||||
type: 'graph-ready';
|
||||
}
|
||||
|
||||
export interface ErrorMessage {
|
||||
type: 'error';
|
||||
message: string;
|
||||
|
|
@ -75,4 +102,9 @@ export interface ErrorMessage {
|
|||
}
|
||||
|
||||
/** Child → parent IPC messages. Shared with the parent-side launcher. */
|
||||
export type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage;
|
||||
export type WorkerMessage =
|
||||
| ProgressMessage
|
||||
| GraphPreparedMessage
|
||||
| GraphReadyMessage
|
||||
| CompleteMessage
|
||||
| ErrorMessage;
|
||||
|
|
|
|||
|
|
@ -74,10 +74,38 @@ process.on('unhandledRejection', (reason: unknown) => {
|
|||
// returned to JS. SIGTERM is retained only for local process shutdown.
|
||||
let cancellationRequested = false;
|
||||
let started = false;
|
||||
const cancellation = new AbortController();
|
||||
let graphVisible: (() => void) | undefined;
|
||||
|
||||
async function waitForGraphVisibility(): Promise<void> {
|
||||
cancellation.signal.throwIfAborted();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
graphVisible = undefined;
|
||||
cancellation.signal.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
reject(cancellation.signal.reason);
|
||||
};
|
||||
graphVisible = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
cancellation.signal.addEventListener('abort', onAbort, { once: true });
|
||||
try {
|
||||
send({ type: 'graph-ready' });
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
function requestWorkerCancellation(source: string): void {
|
||||
if (cancellationRequested) return;
|
||||
cancellationRequested = true;
|
||||
if (claimTerminal()) {
|
||||
cancellation.abort(new Error(`Analysis cancelled (${source})`));
|
||||
if (claimTerminal() && process.connected) {
|
||||
send({ type: 'error', message: `Analysis cancelled (${source})` });
|
||||
}
|
||||
if (!started) {
|
||||
|
|
@ -102,9 +130,14 @@ function exitAfterCancellation(): void {
|
|||
}
|
||||
|
||||
process.on('SIGTERM', () => requestWorkerCancellation('worker received SIGTERM'));
|
||||
process.on('disconnect', () => requestWorkerCancellation('supervisor disconnected'));
|
||||
|
||||
// Listen for parent commands — guarded against re-entry.
|
||||
process.on('message', async (msg: ParentMessage) => {
|
||||
if (msg.type === 'graph-visible') {
|
||||
graphVisible?.();
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'cancel') {
|
||||
requestWorkerCancellation('parent requested cancellation');
|
||||
return;
|
||||
|
|
@ -141,11 +174,16 @@ process.on('message', async (msg: ParentMessage) => {
|
|||
msg.options,
|
||||
{
|
||||
runFullAnalysis: prepared.loaded.analysisModule.runFullAnalysis,
|
||||
continuePreparedAnalysis: prepared.loaded.analysisModule.continuePreparedAnalysis,
|
||||
onGraphPublished: msg.options.graphPublication ? waitForGraphVisibility : undefined,
|
||||
signal: cancellation.signal,
|
||||
assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized,
|
||||
send,
|
||||
claimTerminal,
|
||||
isCancelled: () => cancellationRequested,
|
||||
},
|
||||
prepared.runnerIdentity,
|
||||
msg.type === 'continue' ? msg : undefined,
|
||||
);
|
||||
} catch (error) {
|
||||
if (claimTerminal()) {
|
||||
|
|
|
|||
|
|
@ -9,14 +9,16 @@
|
|||
*/
|
||||
|
||||
import {
|
||||
acquireIndexLock,
|
||||
IndexLockTimeoutError,
|
||||
requireExclusiveIndexLock,
|
||||
sweepStagingArtifacts,
|
||||
type IndexLockHandle,
|
||||
} from '../storage/index-lock.js';
|
||||
import { acquireAnalysisSlotLock } from '../storage/analysis-slot-lock.js';
|
||||
import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js';
|
||||
import { resolveGraphPath } from '../storage/shared-store.js';
|
||||
import { readEmbeddingJobProgress } from '../storage/embedding-job-progress.js';
|
||||
import { readIndexCapabilities } from '../storage/index-capabilities.js';
|
||||
import {
|
||||
reclaimAfterSlotRemoval,
|
||||
removeCheckoutStorage,
|
||||
|
|
@ -29,7 +31,6 @@ import {
|
|||
canonicalizePath,
|
||||
cloneDirBelongsToEntry,
|
||||
loadMeta,
|
||||
saveMeta,
|
||||
listRegisteredRepos,
|
||||
registryPathEquals,
|
||||
type RegistryEntry,
|
||||
|
|
@ -44,10 +45,9 @@ import {
|
|||
import {
|
||||
executeQuery,
|
||||
executePrepared,
|
||||
executeWithReusedStatement,
|
||||
streamQuery,
|
||||
flushWAL,
|
||||
closeLbug,
|
||||
drainAnalysisNativeResources,
|
||||
withLbugDb,
|
||||
isReadOnlyDbError,
|
||||
} from '../core/lbug/lbug-adapter.js';
|
||||
|
|
@ -62,24 +62,13 @@ import {
|
|||
contentRetentionFromMeta,
|
||||
isFullSourceAvailable,
|
||||
} from '../core/content-retention.js';
|
||||
import { LBUG_DIRECTORY } from '../storage/storage-constants.js';
|
||||
import { getFtsDisabledReason, type FtsDisabledReason } from '../core/search/fts-policy.js';
|
||||
import { LocalBackend } from '../mcp/local/local-backend.js';
|
||||
import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { isTerminalJobStatus, JobManager, type AnalyzeJobPartialOutcome } from './analyze-job.js';
|
||||
import { isTerminalJobStatus, JobManager } from './analyze-job.js';
|
||||
import { mountSSEProgress } from './sse-progress.js';
|
||||
import {
|
||||
resolveEmbedRunOutcome,
|
||||
withMeasuredEmbeddingCount,
|
||||
type EmbedRunFinalizeContext,
|
||||
} from './embed-run-outcome.js';
|
||||
import { decideEmbeddingResume, mintInterruptedCheckpoint } from '../core/embedding-checkpoint.js';
|
||||
import {
|
||||
measurePersistedEmbeddingCount,
|
||||
persistedEmbeddingCountOrUndefined,
|
||||
type PersistedEmbeddingCount,
|
||||
} from '../core/embedding-count.js';
|
||||
import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
import { assertString, BadRequestError, createRouteLimiter } from './validation.js';
|
||||
import { parseGrepQuery, GREP_TIME_BUDGET_MS } from './grep-params.js';
|
||||
import { runGrepScanInWorker } from './grep-scan.js';
|
||||
|
|
@ -997,6 +986,8 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
const app = express();
|
||||
app.disable('x-powered-by');
|
||||
const serverStartedAt = Date.now();
|
||||
let shuttingDown = false;
|
||||
const activeEmbeddingRuns = new Set<Promise<void>>();
|
||||
|
||||
// Which upstream hops may set X-Forwarded-*. Process-wide: every route's
|
||||
// req.ip, and so the per-IP rate limiter, resolves through this.
|
||||
|
|
@ -1065,6 +1056,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
const activeRepoPaths = new Set<string>();
|
||||
|
||||
const acquireRepoLock = (repoPath: string): string | null => {
|
||||
if (shuttingDown) return 'Server is shutting down';
|
||||
if (activeRepoPaths.has(repoPath)) {
|
||||
return `Another job is already active for this repository`;
|
||||
}
|
||||
|
|
@ -1152,20 +1144,28 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) {
|
||||
if (clientGone) return null; // client disconnected — stop polling
|
||||
const currentJob = jobManager.getJob(job.id);
|
||||
if (!currentJob || currentJob.status === 'failed') {
|
||||
if (
|
||||
currentJob?.progress.graphPhase === 'ready' ||
|
||||
currentJob?.status === 'complete' ||
|
||||
currentJob?.status === 'failed'
|
||||
) {
|
||||
// A published graph is usable while embeddings are pending or
|
||||
// failed. Re-resolve its owned registry entry before returning.
|
||||
await backend.init();
|
||||
const freshRepos = await listRegisteredRepos({
|
||||
validate: options.validateStorage !== false,
|
||||
});
|
||||
const published = resolveRegisteredRepoEntry(freshRepos, repoName);
|
||||
if (published) return validate(published);
|
||||
if (currentJob.status === 'complete' || currentJob.status === 'failed') return null;
|
||||
}
|
||||
if (!currentJob) {
|
||||
// The job is over and produced no registry entry. This is a
|
||||
// plain "not found", not "still analyzing" — falling through to
|
||||
// the timed-out sentinel here told callers to keep waiting for
|
||||
// a job that had already failed.
|
||||
return null;
|
||||
}
|
||||
if (currentJob.status === 'complete') {
|
||||
await backend.init();
|
||||
const freshRepos = await listRegisteredRepos({
|
||||
validate: options.validateStorage !== false,
|
||||
});
|
||||
return validate(resolveRegisteredRepoEntry(freshRepos, repoName));
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
// Timed out — signal to the caller with a specific message
|
||||
|
|
@ -1270,15 +1270,24 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
// Timed out waiting for an active analysis job
|
||||
if (respondIfAnalysisPending(entry, res)) return;
|
||||
const meta = await loadMeta(entry.storagePath);
|
||||
const [staleness, availability] = await Promise.all([
|
||||
const [staleness, availability, embeddingProgress, capabilities] = await Promise.all([
|
||||
checkStalenessAsync(entry.path, resolveLastCommit(entry, meta)),
|
||||
getSourceAvailability(entry, meta),
|
||||
readEmbeddingJobProgress(entry.storagePath, meta),
|
||||
readIndexCapabilities(entry.storagePath, meta),
|
||||
]);
|
||||
res.json(
|
||||
projectRepoDetail(entry, meta, staleness, {
|
||||
contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta),
|
||||
sourceAvailable: availability.available,
|
||||
}),
|
||||
projectRepoDetail(
|
||||
entry,
|
||||
meta,
|
||||
staleness,
|
||||
{
|
||||
contentRetention: availability.contentRetention ?? contentRetentionFromMeta(meta),
|
||||
sourceAvailable: availability.available,
|
||||
},
|
||||
embeddingProgress,
|
||||
capabilities,
|
||||
),
|
||||
);
|
||||
} catch (err: any) {
|
||||
if (sendStorageRequirementHttp(err, res)) return;
|
||||
|
|
@ -1510,15 +1519,33 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
async () => {
|
||||
let searchResults: any[];
|
||||
let ftsAvailable: boolean | undefined;
|
||||
const metadata = mode === 'bm25' ? null : await loadMeta(entry.storagePath);
|
||||
const snapshot =
|
||||
metadata &&
|
||||
(metadata.graphGeneration || metadata.vectorGeneration || metadata.embeddingJob)
|
||||
? { metadataDirectory: entry.storagePath, metadata, graphPath: lbugPath }
|
||||
: undefined;
|
||||
const semanticReady =
|
||||
mode !== 'bm25' && (await import('../core/embeddings/embedder.js')).isEmbedderReady();
|
||||
const semanticLane = async (execute: typeof executeQuery, text: string, count = 10) => {
|
||||
const { semanticSearch } = await import('../core/embeddings/embedding-pipeline.js');
|
||||
return semanticSearch(
|
||||
execute,
|
||||
text,
|
||||
count,
|
||||
undefined,
|
||||
snapshot,
|
||||
snapshot
|
||||
? async (input) => (await import('../mcp/core/embedder.js')).embedQuery(input)
|
||||
: undefined,
|
||||
);
|
||||
};
|
||||
|
||||
if (mode === 'semantic') {
|
||||
const { isEmbedderReady } = await import('../core/embeddings/embedder.js');
|
||||
if (!isEmbedderReady()) {
|
||||
if (!semanticReady) {
|
||||
return { searchResults: [] as any[], ftsAvailable: undefined };
|
||||
}
|
||||
const { semanticSearch: semSearch } =
|
||||
await import('../core/embeddings/embedding-pipeline.js');
|
||||
searchResults = await semSearch(executeQuery, query, limit);
|
||||
searchResults = await semanticLane(executeQuery, query, limit);
|
||||
// Normalize semantic results to HybridSearchResult shape
|
||||
searchResults = searchResults.map((r: any, i: number) => ({
|
||||
...r,
|
||||
|
|
@ -1536,15 +1563,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
}));
|
||||
} else {
|
||||
// hybrid (default)
|
||||
const { isEmbedderReady } = await import('../core/embeddings/embedder.js');
|
||||
if (isEmbedderReady()) {
|
||||
const { semanticSearch: semSearch } =
|
||||
await import('../core/embeddings/embedding-pipeline.js');
|
||||
if (semanticReady) {
|
||||
searchResults = await hybridSearch(
|
||||
query,
|
||||
limit,
|
||||
executeQuery,
|
||||
semSearch,
|
||||
semanticLane,
|
||||
ftsDisabledReason,
|
||||
);
|
||||
if (ftsDisabledReason) ftsAvailable = false;
|
||||
|
|
@ -2033,7 +2057,6 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
});
|
||||
}
|
||||
})();
|
||||
|
||||
res.status(202).json({ jobId: job.id, status: job.status });
|
||||
} catch (err: any) {
|
||||
if (err.message?.includes('already in progress')) {
|
||||
|
|
@ -2144,266 +2167,86 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
}, EMBED_TIMEOUT_MS);
|
||||
|
||||
// Run embedding pipeline asynchronously
|
||||
(async () => {
|
||||
// Set inside withLbugDb, read after it closes (#2790).
|
||||
let partialRunError: string | undefined;
|
||||
let partialRunDetail: AnalyzeJobPartialOutcome | undefined;
|
||||
// The in-memory repo lock only serializes this server; a CLI analyze
|
||||
// in another process guards the slot with the index lock, so hold it
|
||||
// for the whole embedding write, released in the finally below.
|
||||
const run = (async () => {
|
||||
// Both server writers and other CLI processes serialize on this slot.
|
||||
let slotLock: IndexLockHandle | undefined;
|
||||
try {
|
||||
slotLock = await acquireIndexLock(storagePath, { sweep: false });
|
||||
slotLock = await acquireAnalysisSlotLock(storagePath, { sweep: false });
|
||||
requireExclusiveIndexLock(
|
||||
slotLock,
|
||||
`Cannot acquire the index lock at ${storagePath}; refusing an unlocked embedding run.`,
|
||||
);
|
||||
// This writer cannot recover staged generations. Preserve their
|
||||
// receipts, including malformed ones, before sweeping or writing.
|
||||
const recoveryCheckpoint = (await loadMeta(storagePath))?.embeddingCheckpoint;
|
||||
if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) {
|
||||
embedController.signal.throwIfAborted();
|
||||
const metadata = await loadMeta(storagePath);
|
||||
if (
|
||||
metadata?.embeddingCheckpoint &&
|
||||
Object.hasOwn(metadata.embeddingCheckpoint, 'recovery')
|
||||
)
|
||||
throw new Error(
|
||||
'Cannot generate embeddings: the index checkpoint references staged embeddings. ' +
|
||||
'Run `gitnexus analyze` to recover them first.',
|
||||
'Cannot generate embeddings: the index checkpoint references staged embeddings. Run `gitnexus analyze` to recover them first.',
|
||||
);
|
||||
}
|
||||
const { readPreparedAnalysis } = await import('../core/analysis-publication.js');
|
||||
if (await readPreparedAnalysis(storagePath))
|
||||
throw new Error('A prepared graph awaits publication. Run gitnexus analyze first.');
|
||||
sweepStagingArtifacts(storagePath);
|
||||
// Writes go to the slot's own graph; a shared-store checkout
|
||||
// reading an immutable commit graph (#3352) takes a private copy.
|
||||
if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) {
|
||||
const { resolveHostAnalysisResources } =
|
||||
await import('../core/analysis-host-resources.js');
|
||||
const resources = resolveHostAnalysisResources({
|
||||
includeParsing: false,
|
||||
includeEmbeddings: true,
|
||||
});
|
||||
if (!(await ensurePrivateSharedGraph(storagePath, () => {})))
|
||||
throw new Error('The shared graph this repository reads is gone. Re-run analyze.');
|
||||
}
|
||||
const lbugPath = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const ftsSession = await loadFtsSession(storagePath);
|
||||
let embeddingMeta = ftsSession.meta;
|
||||
// Writable embed still replays a leftover FTS-abort WAL. Refuse
|
||||
// here — doInitLbug only gates the readOnly path, and analyze
|
||||
// writers must still be able to park/rebuild.
|
||||
const lbugPath = resolveGraphPath(storagePath);
|
||||
await assertReadOnlyFtsCrashSafe(lbugPath);
|
||||
await withLbugDb(
|
||||
lbugPath,
|
||||
async () => {
|
||||
const { runEmbeddingPipeline } =
|
||||
await import('../core/embeddings/embedding-pipeline.js');
|
||||
const { resolveEmbeddingIdentity } =
|
||||
await import('../core/embeddings/embedding-identity.js');
|
||||
const embeddingIdentity = resolveEmbeddingIdentity();
|
||||
if (!embeddingMeta) {
|
||||
throw new Error('Repository metadata is missing; run gitnexus analyze first');
|
||||
}
|
||||
const priorCheckpoint = embeddingMeta.embeddingCheckpoint;
|
||||
// The SAME decision the CLI's resume gate makes
|
||||
// (core/embedding-checkpoint.ts). This route used to hard-throw on
|
||||
// any identity mismatch and ignore `attempts` entirely, so a
|
||||
// `'partial'` marker written by `gitnexus analyze` and resumed
|
||||
// here hit exactly the permanent wedge `kind` exists to remove:
|
||||
// two readers of one record disagreeing about the rule it encodes.
|
||||
// No `force`/`--drop-embeddings` equivalent exists on this route,
|
||||
// so the flag options go unset and `'discard'` is unreachable —
|
||||
// it is folded into the abandon arm rather than given an invented
|
||||
// flag. `maxAttempts` is left to the shared default.
|
||||
const resume = priorCheckpoint
|
||||
? decideEmbeddingResume(priorCheckpoint, embeddingIdentity)
|
||||
: undefined;
|
||||
if (resume?.action === 'abort') throw new Error(resume.error);
|
||||
if (resume?.action === 'abandon' || resume?.action === 'discard') {
|
||||
logger.warn({ repo: entry.name }, resume.log);
|
||||
}
|
||||
const forceReembedNodeIds: ReadonlySet<string> =
|
||||
resume?.action === 'resume' ? resume.pendingNodeIds : new Set<string>();
|
||||
const saveEmbeddingCheckpoint = async (
|
||||
checkpoint: {
|
||||
nodesProcessed: number;
|
||||
totalNodes: number;
|
||||
chunksProcessed: number;
|
||||
},
|
||||
pendingNodeIds: string[],
|
||||
embeddings?: PersistedEmbeddingCount,
|
||||
): Promise<void> => {
|
||||
// tri-review NEW-2: re-read immediately before writing (mirrors
|
||||
// the pattern in run-analyze.ts's --repair-fts stamp) instead of
|
||||
// spreading the stale `embeddingMeta` snapshot captured once at
|
||||
// job start. This job can run up to EMBED_TIMEOUT_MS (30 min);
|
||||
// without a fresh read, a concurrent writer's update (e.g. a
|
||||
// --repair-fts capability stamp) would be silently reverted on
|
||||
// every checkpoint save for the job's whole lifetime.
|
||||
const latestMeta = (await loadMeta(storagePath)) ?? embeddingMeta;
|
||||
// `stats.embeddings` only moves when the caller MEASURED the
|
||||
// live count (the post-flush `onCheckpoint`). The window-start
|
||||
// callback measures nothing and passes nothing: restating the
|
||||
// old count there would re-publish a stale number and clobber
|
||||
// what a preceding `onCheckpoint` just wrote (same split as
|
||||
// run-analyze.ts's checkpoint writer).
|
||||
embeddingMeta = withMeasuredEmbeddingCount(
|
||||
{
|
||||
...latestMeta,
|
||||
// In flight ⇒ `kind: 'interrupted'` (embedding-checkpoint.ts).
|
||||
embeddingCheckpoint: mintInterruptedCheckpoint(
|
||||
embeddingIdentity,
|
||||
checkpoint,
|
||||
pendingNodeIds,
|
||||
),
|
||||
await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS);
|
||||
const { runAnalysisEmbeddingPhase } =
|
||||
await import('../core/analysis-embedding-phase.js');
|
||||
await runAnalysisEmbeddingPhase({
|
||||
metadataDirectory: storagePath,
|
||||
graphPath: lbugPath,
|
||||
resources,
|
||||
nodeLimit: 0,
|
||||
expectedGeneration: metadata?.graphGeneration?.id,
|
||||
signal: embedController.signal,
|
||||
callbacks: {
|
||||
onGraphPublished: async () => {
|
||||
await drainAnalysisNativeResources();
|
||||
await backend.init();
|
||||
},
|
||||
onProgress: (phase, percent, message, details) => {
|
||||
embedJobManager.updateJob(job.id, {
|
||||
progress: {
|
||||
...projectAnalysisPhaseProgress(details),
|
||||
phase:
|
||||
phase === 'embeddings'
|
||||
? details?.embeddingProgress?.phase === 'indexing'
|
||||
? 'indexing'
|
||||
: 'embedding'
|
||||
: phase,
|
||||
percent,
|
||||
message,
|
||||
},
|
||||
embeddings,
|
||||
);
|
||||
await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS);
|
||||
await saveMeta(storagePath, embeddingMeta);
|
||||
};
|
||||
/**
|
||||
* Count the persisted rows, or report the answer never arrived.
|
||||
* The TRI-STATE is carried to the fold rather than collapsed here:
|
||||
* `unknown` is not 0, and only the fold knows what to carry
|
||||
* forward instead (core/embedding-count.ts).
|
||||
*/
|
||||
const countPersistedEmbeddings = async (): Promise<PersistedEmbeddingCount> => {
|
||||
const counted = await measurePersistedEmbeddingCount(executeQuery);
|
||||
if (counted.kind === 'unknown') {
|
||||
logger.warn(
|
||||
{ reason: counted.reason },
|
||||
'[embed] could not count persisted embeddings; leaving stats.embeddings untouched',
|
||||
);
|
||||
}
|
||||
return counted;
|
||||
};
|
||||
// Fetch existing content hashes for incremental embedding.
|
||||
// Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling.
|
||||
const { fetchExistingEmbeddingHashes } =
|
||||
await import('../core/lbug/lbug-adapter.js');
|
||||
const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery);
|
||||
if (existingEmbeddings && existingEmbeddings.size > 0) {
|
||||
console.log(
|
||||
`[embed] ${existingEmbeddings.size} nodes already embedded — incremental run with content-hash comparison`,
|
||||
);
|
||||
}
|
||||
const pipelineResult = await runEmbeddingPipeline(
|
||||
executeQuery,
|
||||
executeWithReusedStatement,
|
||||
(p) => {
|
||||
embedJobManager.updateJob(job.id, {
|
||||
progress: {
|
||||
// `ready` maps to 'finalizing', NOT 'complete' (#2790).
|
||||
// The pipeline emits `ready`/100% unconditionally before
|
||||
// returning — including when it dropped nodes to endpoint
|
||||
// failures — and the route has not measured the index or
|
||||
// decided the outcome yet, so 'complete' here would make
|
||||
// the job record contradict itself (`status: 'analyzing'`,
|
||||
// `progress.phase: 'complete'`).
|
||||
phase:
|
||||
p.phase === 'ready'
|
||||
? 'finalizing'
|
||||
: p.phase === 'error'
|
||||
? 'failed'
|
||||
: p.phase,
|
||||
percent: p.percent,
|
||||
message:
|
||||
p.phase === 'loading-model'
|
||||
? 'Loading embedding model...'
|
||||
: p.phase === 'embedding'
|
||||
? `Embedding nodes (${p.percent}%)...`
|
||||
: p.phase === 'indexing'
|
||||
? 'Creating vector index...'
|
||||
: p.phase === 'ready'
|
||||
? 'Finalizing embeddings...'
|
||||
: `${p.phase} (${p.percent}%)`,
|
||||
},
|
||||
});
|
||||
},
|
||||
{}, // config: use defaults
|
||||
undefined, // skipNodeIds
|
||||
existingEmbeddings,
|
||||
{
|
||||
source: {
|
||||
repoPath: entry.path,
|
||||
fileHashes: embeddingMeta.fileHashes,
|
||||
contentRetention: contentRetentionFromMeta(embeddingMeta),
|
||||
},
|
||||
signal: embedController.signal,
|
||||
forceReembedNodeIds,
|
||||
onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => {
|
||||
await saveEmbeddingCheckpoint(checkpoint, nodeIds);
|
||||
},
|
||||
onCheckpoint: async (checkpoint) => {
|
||||
// Count AFTER the flush, so the number describes rows that
|
||||
// are durable rather than rows still pending in the WAL.
|
||||
await flushWAL();
|
||||
await saveEmbeddingCheckpoint(
|
||||
checkpoint,
|
||||
[],
|
||||
await countPersistedEmbeddings(),
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Flush WAL so subsequent /api/search requests see the new
|
||||
// embeddings immediately (#1149). In the CLI path closeLbug()
|
||||
// handles this during process exit, but the server keeps the
|
||||
// connection open for other routes — a CHECKPOINT is enough.
|
||||
await flushWAL();
|
||||
// Measure inside withLbugDb, after the flush and while the
|
||||
// connection is still open — this is the route's only chance to
|
||||
// stamp `stats.embeddings` (embed-run-outcome.ts). A partial run
|
||||
// gets the same stamp: an honest count of a partial index is what
|
||||
// makes it survivable.
|
||||
const measuredEmbeddings = await countPersistedEmbeddings();
|
||||
// Same re-read-before-write reasoning as saveEmbeddingCheckpoint
|
||||
// above — and the outcome decision reads it too: its
|
||||
// `embeddingCheckpoint` is the marker this run's own mid-run
|
||||
// writer saved, which is the only record of the work when the
|
||||
// count query could not answer.
|
||||
const finalMeta = (await loadMeta(storagePath)) ?? embeddingMeta;
|
||||
const finalizeContext: EmbedRunFinalizeContext = {
|
||||
measuredEmbeddings: persistedEmbeddingCountOrUndefined(measuredEmbeddings),
|
||||
onDisk: finalMeta,
|
||||
// The marker the job STARTED from — `finalMeta`'s has since been
|
||||
// overwritten by the in-flight writer, so only this one carries
|
||||
// the `'partial'` attempt chain.
|
||||
resumedFrom: priorCheckpoint,
|
||||
};
|
||||
const outcome = resolveEmbedRunOutcome(
|
||||
embeddingIdentity,
|
||||
pipelineResult,
|
||||
finalizeContext,
|
||||
);
|
||||
partialRunError = outcome.error;
|
||||
partialRunDetail = outcome.partial;
|
||||
embeddingMeta = withMeasuredEmbeddingCount(
|
||||
{ ...finalMeta, embeddingCheckpoint: outcome.checkpoint },
|
||||
measuredEmbeddings,
|
||||
);
|
||||
await requireRegisteredStoragePath(entry, STATUS_STORAGE_REQUIREMENTS);
|
||||
await saveMeta(storagePath, embeddingMeta);
|
||||
});
|
||||
},
|
||||
onLog: (message) => logger.info({ repo: entry.name }, message),
|
||||
},
|
||||
skipFtsOption(ftsSession.skipFts),
|
||||
);
|
||||
|
||||
// Don't overwrite 'failed' if the job was cancelled while the pipeline was running
|
||||
});
|
||||
await backend.init();
|
||||
const current = embedJobManager.getJob(job.id);
|
||||
if (!current || current.status !== 'failed') {
|
||||
// The ONLY terminal event for the job, on both branches — nothing
|
||||
// earlier maps to a terminal status, and `updateJob` synthesizes
|
||||
// exactly one event per terminal status (#2264 P3). The explicit
|
||||
// `progress` keeps the record self-consistent for a poller reading
|
||||
// `progress.phase` (#2790).
|
||||
embedJobManager.updateJob(
|
||||
job.id,
|
||||
partialRunError === undefined
|
||||
? {
|
||||
status: 'complete',
|
||||
progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' },
|
||||
}
|
||||
: {
|
||||
status: 'failed',
|
||||
error: partialRunError,
|
||||
// Lets a client separate "retry these N nodes" from "the
|
||||
// run produced nothing" without a new status member.
|
||||
partial: partialRunDetail,
|
||||
progress: { phase: 'failed', percent: 100, message: partialRunError },
|
||||
},
|
||||
);
|
||||
}
|
||||
if (current && !isTerminalJobStatus(current.status))
|
||||
embedJobManager.updateJob(job.id, {
|
||||
status: 'complete',
|
||||
progress: { phase: 'complete', percent: 100, message: 'Embeddings complete' },
|
||||
});
|
||||
} catch (err: any) {
|
||||
const current = embedJobManager.getJob(job.id);
|
||||
// Provider causes may contain response text. Log only the public
|
||||
// message, as the CLI does, without pino's recursive Error serializer.
|
||||
logger.error(
|
||||
{ reason: err.message || 'Embedding generation failed', repo: entry.name },
|
||||
'Embedding generation failed',
|
||||
);
|
||||
if (!current || current.status !== 'failed') {
|
||||
embedJobManager.updateJob(job.id, {
|
||||
status: 'failed',
|
||||
|
|
@ -2416,6 +2259,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
releaseRepoLock(repoLockPath);
|
||||
}
|
||||
})();
|
||||
activeEmbeddingRuns.add(run);
|
||||
// Observe both outcomes without creating an unhandled finally promise.
|
||||
void run.then(
|
||||
() => activeEmbeddingRuns.delete(run),
|
||||
() => activeEmbeddingRuns.delete(run),
|
||||
);
|
||||
|
||||
res.status(202).json({ jobId: job.id, status: 'analyzing' });
|
||||
} catch (err: any) {
|
||||
|
|
@ -2554,11 +2403,17 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
// destination is `sync: false` (buffered); `flushLoggerSync()` before
|
||||
// `process.exit` so records emitted during cleanup reach stderr.
|
||||
const shutdown = async () => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
console.log('\nShutting down...');
|
||||
updateController.stop();
|
||||
server.close();
|
||||
jobManager.dispose();
|
||||
embedJobManager.dispose();
|
||||
// Abort requests are cooperative. Keep the ordinary writer locks and
|
||||
// native handles alive until each in-process embedding run has settled.
|
||||
await Promise.allSettled(activeEmbeddingRuns);
|
||||
await jobManager.waitForChildren();
|
||||
await cleanupMcp();
|
||||
await closeLbug();
|
||||
await backend.disconnect();
|
||||
|
|
@ -2576,12 +2431,10 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
// Worker-thread transports (pino-pretty under TTY) handle their own
|
||||
// flush on process exit in v10. `pino.final` was removed in v10
|
||||
// because the new transport architecture made it unnecessary.
|
||||
let shuttingDown = false;
|
||||
process.on('uncaughtException', (err) => {
|
||||
logger.error({ err }, 'GitNexus uncaughtException');
|
||||
flushLoggerSync();
|
||||
if (!shuttingDown) {
|
||||
shuttingDown = true;
|
||||
shutdown().catch(() => {});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
} from './analyze-job.js';
|
||||
import { publicRepoId } from './public-repo-id.js';
|
||||
import { escapeRegExp } from './validation.js';
|
||||
import { projectAnalysisPhaseProgress } from '../core/analysis-progress.js';
|
||||
|
||||
export interface OpsJobView {
|
||||
id: string;
|
||||
|
|
@ -227,11 +228,14 @@ export const redactPublicText = (text: string, known?: Array<string | undefined>
|
|||
export const publicOpsProgress = (
|
||||
progress: AnalyzeJobProgress,
|
||||
known?: Array<string | undefined>,
|
||||
): AnalyzeJobProgress => ({
|
||||
phase: progress.phase,
|
||||
percent: progress.percent,
|
||||
message: redactPublicText(progress.message, known),
|
||||
});
|
||||
): AnalyzeJobProgress => {
|
||||
return {
|
||||
phase: progress.phase,
|
||||
percent: progress.percent,
|
||||
message: redactPublicText(progress.message, known),
|
||||
...projectAnalysisPhaseProgress(progress),
|
||||
};
|
||||
};
|
||||
|
||||
export const serializeOpsJob = (
|
||||
job: AnalyzeJob,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ import {
|
|||
import type { ContentRetention, RepoMeta } from '../storage/repo-meta.js';
|
||||
import type { RegistryEntry } from '../storage/repo-manager.js';
|
||||
import { publicRepoId } from './public-repo-id.js';
|
||||
import { getGraphIncompleteReasons, getIndexIncompleteReasons } from '../core/index-freshness.js';
|
||||
import type { EmbeddingJobProgress } from '../core/analysis-progress.js';
|
||||
|
||||
/** Retention + checkout facts computed by the route (see getSourceAvailability). */
|
||||
export interface RepoProjectionSource {
|
||||
|
|
@ -85,6 +87,8 @@ export const projectRepoDetail = (
|
|||
meta: RepoMeta | null | undefined,
|
||||
staleness: StalenessInfo,
|
||||
source: RepoProjectionSource,
|
||||
embeddingProgress?: EmbeddingJobProgress,
|
||||
capabilities?: RepoMeta['capabilities'],
|
||||
) => ({
|
||||
name: entry.name,
|
||||
repoPath: entry.path,
|
||||
|
|
@ -95,6 +99,10 @@ export const projectRepoDetail = (
|
|||
branch: meta?.branch ?? entry.branch,
|
||||
contentRetention: source.contentRetention,
|
||||
sourceAvailable: source.sourceAvailable,
|
||||
incompleteReasons: getIndexIncompleteReasons(meta),
|
||||
graphIncompleteReasons: getGraphIncompleteReasons(meta),
|
||||
embeddingProgress,
|
||||
capabilities,
|
||||
...stalenessField(staleness),
|
||||
});
|
||||
|
||||
|
|
|
|||
134
gitnexus/src/storage/analysis-artifact-cleanup.ts
Normal file
134
gitnexus/src/storage/analysis-artifact-cleanup.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/** Reclaim only unreferenced owned artifacts under the ordinary slot lock. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { RepoMeta } from './repo-meta.js';
|
||||
import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js';
|
||||
import { readEmbeddingArtifactOwners } from './embedding-artifact.js';
|
||||
import { readLegacyEmbeddingSource } from './legacy-embedding-source.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
const GRAPH = /^lbug\.generation\.[a-f0-9]{32}$/;
|
||||
const ATTEMPT_METADATA = /^analysis-meta-[a-f0-9]{32}\.json$/;
|
||||
const ID = /^[a-f0-9]{64}$/;
|
||||
|
||||
/** Keep the published and previous graph, all model spaces, pending jobs, and
|
||||
* their paid reuse sources. Windows busy files stay in place for the next run. */
|
||||
export async function reclaimAnalysisArtifacts(
|
||||
directory: string,
|
||||
metadata: RepoMeta,
|
||||
previousGraph?: string,
|
||||
log?: (message: string) => void,
|
||||
): Promise<void> {
|
||||
const warn = log ?? ((message: string) => logger.warn(message));
|
||||
try {
|
||||
const root = await embeddingJobDirectory(directory);
|
||||
const entries = await fs.readdir(root, { withFileTypes: true });
|
||||
if (entries.some((entry) => entry.name === 'analysis-publication.json')) return;
|
||||
const graphs = new Set<string>();
|
||||
const jobs = new Set<string>();
|
||||
const legacy = new Set<string>();
|
||||
const keepGraph = (file: string | undefined) => {
|
||||
if (file && GRAPH.test(file)) graphs.add(file);
|
||||
};
|
||||
const keepJob = (id: string | undefined) => {
|
||||
if (!id) return;
|
||||
if (!ID.test(id)) throw new Error('Invalid referenced embedding identity');
|
||||
jobs.add(id);
|
||||
};
|
||||
keepGraph(metadata.graphPath);
|
||||
keepGraph(previousGraph);
|
||||
if (metadata.legacyVectorSource) legacy.add(metadata.legacyVectorSource);
|
||||
for (const reference of [metadata.embeddingJob, metadata.vectorGeneration]) {
|
||||
if (!reference) continue;
|
||||
keepJob(reference.identity);
|
||||
keepJob(reference.artifactIdentity);
|
||||
}
|
||||
for (const owner of await readEmbeddingArtifactOwners(root)) {
|
||||
keepJob(owner.artifactIdentity);
|
||||
keepJob(owner.jobIdentity);
|
||||
keepJob(owner.fallbackArtifact);
|
||||
if (owner.phase === 'pending') keepGraph(`lbug.generation.${owner.graphGeneration}`);
|
||||
if (owner.fallbackLegacy) legacy.add(owner.fallbackLegacy);
|
||||
}
|
||||
// Validate every reference before the first deletion, including retired
|
||||
// legacy graphs that still contain the only copy of paid vectors.
|
||||
for (const identity of legacy) {
|
||||
const source = await readLegacyEmbeddingSource(root, identity);
|
||||
keepGraph(source.sourceFile);
|
||||
}
|
||||
const embeddingRoot = path.join(root, 'embeddings');
|
||||
let jobEntries: string[] = [];
|
||||
try {
|
||||
await embeddingJobDirectory(embeddingRoot);
|
||||
jobEntries = await fs.readdir(embeddingRoot);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
const orphanJobs: Array<{ directory: string; receiptName: string }> = [];
|
||||
for (const name of jobEntries) {
|
||||
if (typeof name !== 'string' || !ID.test(name) || jobs.has(name)) continue;
|
||||
const job = await embeddingJobDirectory(path.join(embeddingRoot, name));
|
||||
for (const receiptName of ['job.json', 'artifact.json', 'retained.json']) {
|
||||
try {
|
||||
const receipt = (await readEmbeddingJobRecord(job, receiptName)) as {
|
||||
version?: number;
|
||||
identity?: string;
|
||||
};
|
||||
// Older retained copies have no identity in their receipt. Keep
|
||||
// those unverifiable directories, as with an interrupted bootstrap.
|
||||
if (receiptName === 'retained.json' && receipt.identity === undefined) break;
|
||||
if (receipt.version !== 1 || receipt.identity !== name)
|
||||
throw new Error('Unreferenced embedding artifact has unknown ownership');
|
||||
orphanJobs.push({ directory: job, receiptName });
|
||||
break;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
const remove = async (file: string, recursive = false): Promise<boolean> => {
|
||||
try {
|
||||
await fs.rm(file, { force: true, recursive });
|
||||
return true;
|
||||
} catch (error) {
|
||||
warn(
|
||||
`Retained busy analysis artifact ${path.basename(file)}; cleanup will retry (${(error as Error).message}).`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
// Without a publication receipt, no attempt metadata is needed for recovery.
|
||||
for (const entry of entries)
|
||||
if (entry.isFile() && ATTEMPT_METADATA.test(entry.name))
|
||||
await remove(path.join(root, entry.name));
|
||||
// A busy sidecar may outlive its base graph; include it in the next sweep.
|
||||
const candidates = new Set(
|
||||
entries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name.replace(/\.(wal|shadow)$/, '')),
|
||||
);
|
||||
for (const graph of candidates) {
|
||||
if (!GRAPH.test(graph) || graphs.has(graph)) continue;
|
||||
if (await remove(path.join(root, graph)))
|
||||
for (const suffix of ['.wal', '.shadow']) await remove(path.join(root, graph + suffix));
|
||||
}
|
||||
for (const { directory: job, receiptName } of orphanJobs) {
|
||||
// Keep the ownership receipt until every payload is gone. Recursive
|
||||
// removal of the whole directory can delete job.json before a busy
|
||||
// native file fails on Windows, making the next cleanup unverifiable.
|
||||
let removed = true;
|
||||
for (const name of await fs.readdir(job)) {
|
||||
if (name === receiptName) continue;
|
||||
if (!(await remove(path.join(job, name), true))) {
|
||||
removed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (removed && (await remove(path.join(job, receiptName)))) await fs.rmdir(job);
|
||||
}
|
||||
} catch (error) {
|
||||
warn(
|
||||
`Retained analysis artifacts because ownership could not be verified (${(error as Error).message}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
53
gitnexus/src/storage/analysis-slot-lock.ts
Normal file
53
gitnexus/src/storage/analysis-slot-lock.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import path from 'node:path';
|
||||
import {
|
||||
acquireIndexLock,
|
||||
requireExclusiveIndexLock,
|
||||
type AcquireOptions,
|
||||
type IndexLockHandle,
|
||||
} from './index-lock.js';
|
||||
import { storeRootOfCheckoutSlot } from './shared-store.js';
|
||||
import { getGlobalDir } from './global-dir.js';
|
||||
|
||||
/** File-backed admission locks must outlive deletion of both member and store.
|
||||
* Keep their empty parent directories: deleting a held gate lets a second
|
||||
* process acquire a different lock at the same pathname. */
|
||||
export function sharedMemberGatePath(storeRoot: string, member: string): string {
|
||||
// Store keys are unique within GITNEXUS_HOME. Keep the names as path
|
||||
// components so filesystem aliases and Windows casing share the same gate.
|
||||
return path.join(
|
||||
getGlobalDir(),
|
||||
'locks',
|
||||
'shared-members',
|
||||
path.basename(storeRoot),
|
||||
path.basename(member),
|
||||
);
|
||||
}
|
||||
|
||||
/** Admit flat and branch writers under the member gate used by shared-store GC.
|
||||
* Once admitted, their index locks protect files without serializing siblings.
|
||||
* GC holds the persistent gate through member, graph, and store deletion. */
|
||||
export async function acquireAnalysisSlotLock(
|
||||
directory: string,
|
||||
options: AcquireOptions = {},
|
||||
): Promise<IndexLockHandle> {
|
||||
const slot = path.resolve(directory);
|
||||
const branches = path.dirname(slot);
|
||||
const member = path.basename(branches) === 'branches' ? path.dirname(branches) : slot;
|
||||
const storeRoot = storeRootOfCheckoutSlot(member);
|
||||
if (!storeRoot) return acquireIndexLock(directory, options);
|
||||
const gatePath = sharedMemberGatePath(storeRoot, member);
|
||||
const gate = await acquireIndexLock(gatePath, { ...options, sweep: false });
|
||||
try {
|
||||
requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${gatePath}.`);
|
||||
if (slot === member) return await acquireIndexLock(directory, options);
|
||||
const memberLock = await acquireIndexLock(member, { ...options, sweep: false });
|
||||
try {
|
||||
requireExclusiveIndexLock(memberLock, `Cannot acquire the shared member lock at ${member}.`);
|
||||
return await acquireIndexLock(directory, options);
|
||||
} finally {
|
||||
memberLock.release();
|
||||
}
|
||||
} finally {
|
||||
gate.release();
|
||||
}
|
||||
}
|
||||
304
gitnexus/src/storage/embedding-artifact.ts
Normal file
304
gitnexus/src/storage/embedding-artifact.ts
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
/** Directly keyed ownership of one pending artifact per compatible input space.
|
||||
* Caller holds the ordinary index lock. No model, native pool or history scan. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
createEmbeddingJobDirectories,
|
||||
embeddingJobDirectory,
|
||||
readEmbeddingJobRecord,
|
||||
writeEmbeddingJobRecord,
|
||||
syncEmbeddingJobDirectory,
|
||||
} from './embedding-job.js';
|
||||
|
||||
export interface EmbeddingArtifactSpace {
|
||||
vectorIdentity: string;
|
||||
inputIdentity: string;
|
||||
dimensions: number;
|
||||
schema: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingArtifactOwner {
|
||||
artifactIdentity: string;
|
||||
jobIdentity: string;
|
||||
graphGeneration: string;
|
||||
sourceIdentity: string;
|
||||
manifestDigest: string;
|
||||
phase: 'pending' | 'sealed';
|
||||
revalidateInputs: boolean;
|
||||
/** At most one completed, read-only source for bounded group reuse. */
|
||||
fallbackArtifact?: string;
|
||||
/** Alternative sole read-only fallback during first legacy migration. */
|
||||
fallbackLegacy?: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingArtifactRecord {
|
||||
version: 1;
|
||||
spaceKey: string;
|
||||
space: EmbeddingArtifactSpace;
|
||||
owner: EmbeddingArtifactOwner;
|
||||
}
|
||||
|
||||
/** Read every model space before reclamation. An interrupted ownership change
|
||||
* retains both sides; malformed/unreadable receipts abort the entire sweep. */
|
||||
export async function readEmbeddingArtifactOwners(root: string): Promise<EmbeddingArtifactOwner[]> {
|
||||
const parent = path.join(root, 'embeddings', 'spaces');
|
||||
try {
|
||||
await fs.lstat(parent);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
|
||||
throw error;
|
||||
}
|
||||
await embeddingJobDirectory(path.dirname(parent));
|
||||
await embeddingJobDirectory(parent);
|
||||
const owners: EmbeddingArtifactOwner[] = [];
|
||||
for (const entry of await fs.readdir(parent, { withFileTypes: true })) {
|
||||
if (!DIGEST.test(entry.name))
|
||||
throw new Error('Unexpected embedding space entry; artifacts retained');
|
||||
const directory = await embeddingJobDirectory(path.join(parent, entry.name));
|
||||
const add = (value: unknown) => {
|
||||
const raw = value as EmbeddingArtifactRecord;
|
||||
const validated = record(raw, raw?.space);
|
||||
if (validated.spaceKey !== entry.name) throw new Error('Embedding space directory mismatch');
|
||||
owners.push(validated.owner);
|
||||
};
|
||||
const current = await optionalRecord(directory, 'space.json');
|
||||
if (current !== null) add(current);
|
||||
const intent = (await optionalRecord(directory, 'rebase.json')) as {
|
||||
version: number;
|
||||
before: unknown;
|
||||
after: unknown;
|
||||
} | null;
|
||||
if (intent !== null) {
|
||||
if (intent.version !== 1 || intent.before === undefined || !intent.after)
|
||||
throw new Error('Invalid embedding rebase intent; artifacts retained');
|
||||
if (intent.before !== null) add(intent.before);
|
||||
add(intent.after);
|
||||
}
|
||||
if (current === null && intent === null && (await fs.readdir(directory)).length)
|
||||
throw new Error('Embedding space has no ownership receipt');
|
||||
}
|
||||
return owners;
|
||||
}
|
||||
|
||||
type ValidateOwner = (owner: EmbeddingArtifactOwner) => Promise<void>;
|
||||
const DIGEST = /^[a-f0-9]{64}$/;
|
||||
|
||||
export function embeddingArtifactSpaceKey(space: EmbeddingArtifactSpace): string {
|
||||
if (
|
||||
!DIGEST.test(space.vectorIdentity) ||
|
||||
!DIGEST.test(space.inputIdentity) ||
|
||||
!Number.isSafeInteger(space.dimensions) ||
|
||||
space.dimensions < 1 ||
|
||||
typeof space.schema !== 'string' ||
|
||||
!space.schema.length ||
|
||||
space.schema.length > 256
|
||||
)
|
||||
throw new Error('Invalid embedding artifact space');
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify([
|
||||
1,
|
||||
space.vectorIdentity,
|
||||
space.inputIdentity,
|
||||
space.dimensions,
|
||||
space.schema,
|
||||
]),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function ownerRecord(value: unknown): EmbeddingArtifactOwner {
|
||||
const owner = value as EmbeddingArtifactOwner;
|
||||
if (
|
||||
!owner ||
|
||||
typeof owner !== 'object' ||
|
||||
!DIGEST.test(owner.artifactIdentity) ||
|
||||
!DIGEST.test(owner.jobIdentity) ||
|
||||
!DIGEST.test(owner.sourceIdentity) ||
|
||||
!DIGEST.test(owner.manifestDigest) ||
|
||||
!/^[a-f0-9]{32}$/.test(owner.graphGeneration) ||
|
||||
!['pending', 'sealed'].includes(owner.phase) ||
|
||||
typeof owner.revalidateInputs !== 'boolean' ||
|
||||
(owner.fallbackArtifact !== undefined &&
|
||||
(!DIGEST.test(owner.fallbackArtifact) ||
|
||||
owner.fallbackArtifact === owner.artifactIdentity)) ||
|
||||
(owner.fallbackLegacy !== undefined &&
|
||||
(!DIGEST.test(owner.fallbackLegacy) || owner.fallbackArtifact !== undefined))
|
||||
)
|
||||
throw new Error('Invalid embedding artifact ownership; artifacts retained');
|
||||
return {
|
||||
artifactIdentity: owner.artifactIdentity,
|
||||
jobIdentity: owner.jobIdentity,
|
||||
graphGeneration: owner.graphGeneration,
|
||||
sourceIdentity: owner.sourceIdentity,
|
||||
manifestDigest: owner.manifestDigest,
|
||||
phase: owner.phase,
|
||||
revalidateInputs: owner.revalidateInputs,
|
||||
...(owner.fallbackArtifact === undefined ? {} : { fallbackArtifact: owner.fallbackArtifact }),
|
||||
...(owner.fallbackLegacy === undefined ? {} : { fallbackLegacy: owner.fallbackLegacy }),
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown, space: EmbeddingArtifactSpace): EmbeddingArtifactRecord {
|
||||
const input = value as EmbeddingArtifactRecord;
|
||||
const spaceKey = embeddingArtifactSpaceKey(space);
|
||||
if (
|
||||
!input ||
|
||||
input.version !== 1 ||
|
||||
input.spaceKey !== spaceKey ||
|
||||
!input.space ||
|
||||
embeddingArtifactSpaceKey(input.space) !== spaceKey
|
||||
)
|
||||
throw new Error('Embedding artifact space mismatch; artifacts retained');
|
||||
return {
|
||||
version: 1,
|
||||
spaceKey,
|
||||
space: {
|
||||
vectorIdentity: space.vectorIdentity,
|
||||
inputIdentity: space.inputIdentity,
|
||||
dimensions: space.dimensions,
|
||||
schema: space.schema,
|
||||
},
|
||||
owner: ownerRecord(input.owner),
|
||||
};
|
||||
}
|
||||
|
||||
async function spaceDirectory(
|
||||
root: string,
|
||||
space: EmbeddingArtifactSpace,
|
||||
create = true,
|
||||
): Promise<string> {
|
||||
let directory = await embeddingJobDirectory(root);
|
||||
const parts = ['embeddings', 'spaces', embeddingArtifactSpaceKey(space)];
|
||||
if (create) return createEmbeddingJobDirectories(directory, parts);
|
||||
for (const part of parts) directory = await embeddingJobDirectory(path.join(directory, part));
|
||||
return directory;
|
||||
}
|
||||
|
||||
async function optionalRecord(directory: string, name: string): Promise<unknown | null> {
|
||||
try {
|
||||
return await readEmbeddingJobRecord(directory, name);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const same = (a: EmbeddingArtifactRecord | null, b: EmbeddingArtifactRecord | null): boolean =>
|
||||
JSON.stringify(a) === JSON.stringify(b);
|
||||
|
||||
function validateTransition(
|
||||
before: EmbeddingArtifactRecord | null,
|
||||
after: EmbeddingArtifactRecord,
|
||||
): void {
|
||||
if (!before) return;
|
||||
const a = before.owner,
|
||||
b = after.owner;
|
||||
if (a.artifactIdentity !== b.artifactIdentity) {
|
||||
if (
|
||||
a.phase !== 'sealed' ||
|
||||
b.phase !== 'pending' ||
|
||||
b.fallbackArtifact !== a.artifactIdentity ||
|
||||
b.fallbackLegacy !== undefined ||
|
||||
b.artifactIdentity === a.fallbackArtifact
|
||||
)
|
||||
throw new Error('Pending paid artifacts must be retained until sealed');
|
||||
return;
|
||||
}
|
||||
if (a.jobIdentity !== b.jobIdentity) {
|
||||
if (
|
||||
a.phase !== 'pending' ||
|
||||
b.phase !== 'pending' ||
|
||||
!b.revalidateInputs ||
|
||||
a.fallbackArtifact !== b.fallbackArtifact ||
|
||||
a.fallbackLegacy !== b.fallbackLegacy
|
||||
)
|
||||
throw new Error(
|
||||
'Rebased artifact requires current-input validation and its retained fallback',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
a.graphGeneration !== b.graphGeneration ||
|
||||
a.sourceIdentity !== b.sourceIdentity ||
|
||||
a.manifestDigest !== b.manifestDigest ||
|
||||
(a.phase === 'sealed' && b.phase !== 'sealed') ||
|
||||
(b.phase === 'pending' && a.fallbackLegacy !== b.fallbackLegacy) ||
|
||||
(a.revalidateInputs && !b.revalidateInputs)
|
||||
)
|
||||
throw new Error('Embedding artifact job identity or sealed state changed');
|
||||
}
|
||||
|
||||
async function finishIntent(directory: string): Promise<void> {
|
||||
await fs.unlink(path.join(directory, 'rebase.json'));
|
||||
await syncEmbeddingJobDirectory(directory);
|
||||
}
|
||||
|
||||
/** A lagging metadata job pointer is reconciled by the controller BEFORE it
|
||||
* opens this artifact for paid work. The owned receipt names the sole writer.
|
||||
* Recovery completes only the exact before/after transition, never guessing
|
||||
* from an old journal cursor or choosing an artifact by directory enumeration. */
|
||||
export async function readEmbeddingArtifactSpace(
|
||||
root: string,
|
||||
space: EmbeddingArtifactSpace,
|
||||
validateOwner: ValidateOwner,
|
||||
): Promise<EmbeddingArtifactRecord | null> {
|
||||
let directory: string;
|
||||
try {
|
||||
directory = await spaceDirectory(root, space, false);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
const currentRaw = await optionalRecord(directory, 'space.json');
|
||||
let current = currentRaw === null ? null : record(currentRaw, space);
|
||||
const intent = (await optionalRecord(directory, 'rebase.json')) as {
|
||||
version?: unknown;
|
||||
before?: unknown;
|
||||
after?: unknown;
|
||||
} | null;
|
||||
if (intent !== null) {
|
||||
if (intent.version !== 1 || intent.before === undefined || !intent.after)
|
||||
throw new Error('Invalid embedding rebase intent; artifacts retained');
|
||||
const before = intent.before === null ? null : record(intent.before, space);
|
||||
const after = record(intent.after, space);
|
||||
validateTransition(before, after);
|
||||
if (!same(current, before) && !same(current, after))
|
||||
throw new Error('Embedding artifact owner superseded its rebase intent; artifacts retained');
|
||||
await validateOwner(after.owner);
|
||||
if (!same(current, after)) await writeEmbeddingJobRecord(directory, 'space.json', after);
|
||||
await finishIntent(directory);
|
||||
current = after;
|
||||
}
|
||||
if (current) await validateOwner(current.owner);
|
||||
return current;
|
||||
}
|
||||
|
||||
/** Manifest creation precedes this durable intent; journal recovery and native
|
||||
* mutation follow it. A crash at either rename can be replayed by read above. */
|
||||
export async function adoptEmbeddingArtifact(
|
||||
root: string,
|
||||
space: EmbeddingArtifactSpace,
|
||||
previous: EmbeddingArtifactRecord | null,
|
||||
owner: EmbeddingArtifactOwner,
|
||||
validateOwner: ValidateOwner,
|
||||
): Promise<EmbeddingArtifactRecord> {
|
||||
const directory = await spaceDirectory(root, space);
|
||||
const before = previous === null ? null : record(previous, space);
|
||||
const after = record(
|
||||
{ version: 1, spaceKey: embeddingArtifactSpaceKey(space), space, owner },
|
||||
space,
|
||||
);
|
||||
const currentRaw = await optionalRecord(directory, 'space.json');
|
||||
const current = currentRaw === null ? null : record(currentRaw, space);
|
||||
if (!same(current, before) || (await optionalRecord(directory, 'rebase.json')) !== null)
|
||||
throw new Error('Embedding artifact ownership changed; recover before adoption');
|
||||
validateTransition(before, after);
|
||||
await validateOwner(after.owner);
|
||||
if (same(before, after)) return after;
|
||||
await writeEmbeddingJobRecord(directory, 'rebase.json', { version: 1, before, after });
|
||||
await writeEmbeddingJobRecord(directory, 'space.json', after);
|
||||
await finishIntent(directory);
|
||||
return after;
|
||||
}
|
||||
503
gitnexus/src/storage/embedding-job-journal.ts
Normal file
503
gitnexus/src/storage/embedding-job-journal.ts
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
/** Compact scheduling journal. The caller owns the checkout's exclusive job lock. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
EmbeddingManifestCursor,
|
||||
EmbeddingManifestReceipt,
|
||||
} from '../core/embeddings/embedding-work-manifest.js';
|
||||
import {
|
||||
embeddingJobDirectory,
|
||||
readEmbeddingJobRecord,
|
||||
syncEmbeddingJobDirectory,
|
||||
writeEmbeddingJobRecord,
|
||||
} from './embedding-job.js';
|
||||
|
||||
export interface EmbeddingUnitIntent {
|
||||
start: EmbeddingManifestCursor;
|
||||
next: EmbeddingManifestCursor;
|
||||
nodes: number;
|
||||
origin: 'manifest' | 'failed';
|
||||
/** Bounded by the unit's 64 nodes; three deterministic rejections exclude a node. */
|
||||
inputFailures?: Record<string, number>;
|
||||
/** Older receipts use raw node IDs. New receipts bound keys independently of ID length. */
|
||||
inputFailureKeyFormat?: 'sha256';
|
||||
}
|
||||
|
||||
export interface EmbeddingJobState {
|
||||
version: 1;
|
||||
identity: string;
|
||||
manifestDigest: string;
|
||||
totalUnits: number;
|
||||
totalNodes: number;
|
||||
manifestBytes: number;
|
||||
cursor: EmbeddingManifestCursor;
|
||||
completedUnits: number;
|
||||
completedNodes: number;
|
||||
reusedUnits: number;
|
||||
reusedNodes: number;
|
||||
failedUnits: number;
|
||||
failedNodes: number;
|
||||
excludedNodes?: number;
|
||||
chunks: number;
|
||||
active: EmbeddingUnitIntent | null;
|
||||
phase: 'embedding' | 'indexing' | 'ready';
|
||||
}
|
||||
|
||||
export type EmbeddingFailureCause = 'provider' | 'input' | 'cancelled';
|
||||
interface FailedUnit {
|
||||
version: 1;
|
||||
identity: string;
|
||||
manifestDigest: string;
|
||||
intent: EmbeddingUnitIntent;
|
||||
cause: EmbeddingFailureCause;
|
||||
attempt: string;
|
||||
}
|
||||
|
||||
const DIGEST = /^[a-f0-9]{64}$/;
|
||||
const FAILED = /^failed-(\d{16})\.json$/;
|
||||
const integer = (value: unknown): value is number =>
|
||||
typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
||||
const filename = (unit: number): string => `failed-${String(unit).padStart(16, '0')}.json`;
|
||||
const clone = <T>(value: T): T => structuredClone(value);
|
||||
|
||||
export const embeddingInputFailureKey = (nodeId: string): string =>
|
||||
createHash('sha256').update(nodeId).digest('hex');
|
||||
|
||||
/** Operational failures (permissions, I/O) must not be treated as lost state. */
|
||||
function damagedRecord(error: unknown): boolean {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
if (code) return code === 'ENOENT' || code === 'ERR_ENCODING_INVALID_ENCODED_DATA';
|
||||
return (
|
||||
error instanceof SyntaxError ||
|
||||
/checksum|Malformed|Invalid|UTF-8|receipt exceeds/i.test(String(error))
|
||||
);
|
||||
}
|
||||
|
||||
function validateIntent(intent: EmbeddingUnitIntent, state: EmbeddingJobState): void {
|
||||
if (
|
||||
!intent ||
|
||||
!intent.start ||
|
||||
!intent.next ||
|
||||
!integer(intent.start.unit) ||
|
||||
intent.start.unit >= state.totalUnits ||
|
||||
!integer(intent.start.offset) ||
|
||||
!integer(intent.next.offset) ||
|
||||
intent.next.offset <= intent.start.offset ||
|
||||
intent.next.offset > state.manifestBytes ||
|
||||
intent.next.unit !== intent.start.unit + 1 ||
|
||||
!integer(intent.nodes) ||
|
||||
intent.nodes < 1 ||
|
||||
intent.nodes > 64 ||
|
||||
!['manifest', 'failed'].includes(intent.origin) ||
|
||||
(intent.inputFailureKeyFormat !== undefined && intent.inputFailureKeyFormat !== 'sha256')
|
||||
)
|
||||
throw new Error('Invalid embedding unit intent');
|
||||
if (
|
||||
intent.inputFailures !== undefined &&
|
||||
(!intent.inputFailures ||
|
||||
typeof intent.inputFailures !== 'object' ||
|
||||
Array.isArray(intent.inputFailures) ||
|
||||
Object.keys(intent.inputFailures).length > intent.nodes ||
|
||||
Object.entries(intent.inputFailures).some(
|
||||
([id, attempts]) =>
|
||||
!id.length ||
|
||||
id.length > 1024 * 1024 ||
|
||||
(intent.inputFailureKeyFormat === 'sha256' && !DIGEST.test(id)) ||
|
||||
!integer(attempts) ||
|
||||
attempts < 1 ||
|
||||
attempts > 3,
|
||||
))
|
||||
)
|
||||
throw new Error('Invalid embedding input rejection history');
|
||||
}
|
||||
|
||||
function validateState(
|
||||
value: unknown,
|
||||
identity: string,
|
||||
manifest: EmbeddingManifestReceipt,
|
||||
): asserts value is EmbeddingJobState {
|
||||
const s = value as EmbeddingJobState;
|
||||
if (
|
||||
!s ||
|
||||
s.version !== 1 ||
|
||||
s.identity !== identity ||
|
||||
s.manifestDigest !== manifest.digest ||
|
||||
s.totalNodes !== manifest.totalNodes ||
|
||||
s.totalUnits !== manifest.totalUnits ||
|
||||
s.manifestBytes !== manifest.bytes ||
|
||||
!s.cursor ||
|
||||
!integer(s.cursor.unit) ||
|
||||
s.cursor.unit > s.totalUnits ||
|
||||
!integer(s.cursor.offset) ||
|
||||
s.cursor.offset > manifest.bytes ||
|
||||
(s.cursor.unit === 0 && s.cursor.offset !== 0) ||
|
||||
(s.cursor.unit === s.totalUnits && s.cursor.offset !== manifest.bytes) ||
|
||||
![
|
||||
s.completedUnits,
|
||||
s.completedNodes,
|
||||
s.reusedUnits,
|
||||
s.reusedNodes,
|
||||
s.failedUnits,
|
||||
s.failedNodes,
|
||||
s.chunks,
|
||||
s.excludedNodes ?? 0,
|
||||
].every(integer) ||
|
||||
s.completedUnits + s.reusedUnits + s.failedUnits > s.totalUnits ||
|
||||
s.completedNodes + s.reusedNodes + s.failedNodes + (s.excludedNodes ?? 0) > s.totalNodes ||
|
||||
!['embedding', 'indexing', 'ready'].includes(s.phase) ||
|
||||
s.active === undefined
|
||||
)
|
||||
throw new Error('Embedding job identity or coverage is invalid; artifacts retained');
|
||||
if (s.active) {
|
||||
validateIntent(s.active, s);
|
||||
const active = s.active;
|
||||
if (
|
||||
active.origin === 'manifest'
|
||||
? active.start.unit !== s.cursor.unit || active.start.offset !== s.cursor.offset
|
||||
: active.next.unit > s.cursor.unit
|
||||
)
|
||||
throw new Error('Embedding job active cursor is inconsistent');
|
||||
}
|
||||
const settled = s.completedUnits + s.reusedUnits + s.failedUnits;
|
||||
if (
|
||||
settled + (s.active?.origin === 'failed' ? 1 : 0) !== s.cursor.unit ||
|
||||
(s.phase !== 'embedding' &&
|
||||
(s.active !== null ||
|
||||
s.failedUnits !== 0 ||
|
||||
settled !== s.totalUnits ||
|
||||
s.completedNodes + s.reusedNodes + (s.excludedNodes ?? 0) !== s.totalNodes))
|
||||
)
|
||||
throw new Error('Embedding job accounting is inconsistent');
|
||||
}
|
||||
|
||||
/** No model, DB, source text, or growing pending-ID list is retained here. */
|
||||
export class EmbeddingJobJournal {
|
||||
private busy = false;
|
||||
private uncertain = false;
|
||||
/** Failed units are attempted at most once per invocation, even if readdir sees a replacement. */
|
||||
readonly attempt = randomUUID();
|
||||
|
||||
private constructor(
|
||||
readonly directory: string,
|
||||
private state: EmbeddingJobState,
|
||||
private readonly manifest: EmbeddingManifestReceipt,
|
||||
) {}
|
||||
|
||||
static async create(
|
||||
directory: string,
|
||||
identity: string,
|
||||
manifest: EmbeddingManifestReceipt,
|
||||
): Promise<EmbeddingJobJournal> {
|
||||
if (!DIGEST.test(identity) || !DIGEST.test(manifest.digest))
|
||||
throw new Error('Invalid embedding job identity');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
try {
|
||||
await fs.lstat(path.join(canonical, 'job.json'));
|
||||
throw new Error('Embedding job already exists');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
const state: EmbeddingJobState = {
|
||||
version: 1,
|
||||
identity,
|
||||
manifestDigest: manifest.digest,
|
||||
totalUnits: manifest.totalUnits,
|
||||
totalNodes: manifest.totalNodes,
|
||||
manifestBytes: manifest.bytes,
|
||||
cursor: { unit: 0, offset: 0 },
|
||||
completedUnits: 0,
|
||||
completedNodes: 0,
|
||||
reusedUnits: 0,
|
||||
reusedNodes: 0,
|
||||
failedUnits: 0,
|
||||
failedNodes: 0,
|
||||
chunks: 0,
|
||||
active: null,
|
||||
phase: 'embedding',
|
||||
};
|
||||
validateState(state, identity, manifest);
|
||||
await writeEmbeddingJobRecord(canonical, 'job.json', state);
|
||||
return new EmbeddingJobJournal(canonical, state, clone(manifest));
|
||||
}
|
||||
|
||||
static async open(
|
||||
directory: string,
|
||||
identity: string,
|
||||
manifest: EmbeddingManifestReceipt,
|
||||
): Promise<EmbeddingJobJournal> {
|
||||
if (!DIGEST.test(identity)) throw new Error('Invalid embedding job identity');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
const state = await readEmbeddingJobRecord(canonical, 'job.json');
|
||||
validateState(state, identity, manifest);
|
||||
return new EmbeddingJobJournal(canonical, state, clone(manifest));
|
||||
}
|
||||
|
||||
snapshot(): EmbeddingJobState {
|
||||
return clone(this.state);
|
||||
}
|
||||
|
||||
/** Recovery requires an independently verified manifest and vector identity.
|
||||
* Preserve damaged receipts; the runner reconstructs coverage only from native
|
||||
* complete groups. A valid receipt for another identity is never discarded.
|
||||
* This also resumes an interruption between preserving and replacing job.json.
|
||||
*/
|
||||
static async recover(
|
||||
directory: string,
|
||||
identity: string,
|
||||
manifest: EmbeddingManifestReceipt,
|
||||
): Promise<EmbeddingJobJournal> {
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
let value: unknown;
|
||||
try {
|
||||
value = await readEmbeddingJobRecord(canonical, 'job.json');
|
||||
} catch (error) {
|
||||
if (!damagedRecord(error)) throw error;
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
const state = value as Partial<EmbeddingJobState>;
|
||||
if (
|
||||
(state.version !== undefined && state.version !== 1) ||
|
||||
(state.identity && state.identity !== identity) ||
|
||||
(state.manifestDigest && state.manifestDigest !== manifest.digest)
|
||||
)
|
||||
throw new Error('Embedding job identity mismatch; recovery refused');
|
||||
let journal: EmbeddingJobJournal | undefined;
|
||||
try {
|
||||
validateState(value, identity, manifest);
|
||||
journal = new EmbeddingJobJournal(canonical, value, clone(manifest));
|
||||
} catch {
|
||||
/* Matching identity but damaged accounting requires reconstruction. */
|
||||
}
|
||||
if (journal) {
|
||||
try {
|
||||
await journal.verifyFailedRecords();
|
||||
return journal;
|
||||
} catch (error) {
|
||||
if (!damagedRecord(error)) throw error;
|
||||
// Complete native node groups survive this scheduling reset. The
|
||||
// runner verifies them before acknowledging without provider calls.
|
||||
}
|
||||
}
|
||||
}
|
||||
const suffix = `.preserved-${randomUUID()}`;
|
||||
try {
|
||||
await fs.rename(path.join(canonical, 'job.json'), path.join(canonical, `job.json${suffix}`));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
await syncEmbeddingJobDirectory(canonical);
|
||||
const entries = await fs.opendir(canonical);
|
||||
for await (const entry of entries) {
|
||||
if (FAILED.test(entry.name))
|
||||
await fs.rename(
|
||||
path.join(canonical, entry.name),
|
||||
path.join(canonical, entry.name + suffix),
|
||||
);
|
||||
}
|
||||
await syncEmbeddingJobDirectory(canonical);
|
||||
return this.create(canonical, identity, manifest);
|
||||
}
|
||||
|
||||
private async mutate(action: (next: EmbeddingJobState) => Promise<void>): Promise<void> {
|
||||
if (this.busy) throw new Error('Concurrent embedding journal mutation');
|
||||
if (this.uncertain)
|
||||
throw new Error('Embedding journal durability uncertain; reopen before retry');
|
||||
this.busy = true;
|
||||
try {
|
||||
const next = clone(this.state);
|
||||
await action(next);
|
||||
validateState(next, next.identity, this.manifest);
|
||||
// A rejected rename/fsync may already have changed durable state. Pure
|
||||
// validation failures before the first write leave the journal usable.
|
||||
this.uncertain = true;
|
||||
await writeEmbeddingJobRecord(this.directory, 'job.json', next);
|
||||
this.state = next;
|
||||
this.uncertain = false;
|
||||
} finally {
|
||||
this.busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
async begin(intent: EmbeddingUnitIntent): Promise<void> {
|
||||
const requested = clone(intent);
|
||||
await this.mutate(async (next) => {
|
||||
if (next.active || next.phase !== 'embedding')
|
||||
throw new Error('Embedding job already active or complete');
|
||||
validateIntent(requested, next);
|
||||
if (requested.origin === 'failed') {
|
||||
const record = await this.readFailed(filename(requested.start.unit));
|
||||
if (
|
||||
record.intent.start.unit !== requested.start.unit ||
|
||||
record.intent.start.offset !== requested.start.offset ||
|
||||
record.intent.next.unit !== requested.next.unit ||
|
||||
record.intent.next.offset !== requested.next.offset ||
|
||||
record.intent.nodes !== requested.nodes
|
||||
)
|
||||
throw new Error('Failed embedding unit cursor mismatch');
|
||||
next.failedUnits--;
|
||||
next.failedNodes -= requested.nodes;
|
||||
requested.inputFailures = record.intent.inputFailures;
|
||||
requested.inputFailureKeyFormat = record.intent.inputFailureKeyFormat;
|
||||
}
|
||||
next.active = requested;
|
||||
});
|
||||
}
|
||||
|
||||
/** Call only after the vector store has verified and checkpointed this unit. */
|
||||
async acknowledge(result: { reusedNodes: number; chunks: number }): Promise<void> {
|
||||
await this.mutate(async (next) => {
|
||||
const active = next.active;
|
||||
const excluded = Object.values(active?.inputFailures ?? {}).filter(
|
||||
(attempts) => attempts >= 3,
|
||||
).length;
|
||||
if (
|
||||
!active ||
|
||||
!integer(result.reusedNodes) ||
|
||||
result.reusedNodes > active.nodes - excluded ||
|
||||
!integer(result.chunks)
|
||||
)
|
||||
throw new Error('Invalid embedding unit acknowledgement');
|
||||
// If the process dies after unlink, the still-active intent owns replay.
|
||||
this.uncertain = true;
|
||||
await fs.rm(path.join(this.directory, filename(active.start.unit)), { force: true });
|
||||
await syncEmbeddingJobDirectory(this.directory);
|
||||
if (result.reusedNodes === active.nodes) next.reusedUnits++;
|
||||
else next.completedUnits++;
|
||||
next.reusedNodes += result.reusedNodes;
|
||||
next.completedNodes += active.nodes - result.reusedNodes - excluded;
|
||||
if (excluded) next.excludedNodes = (next.excludedNodes ?? 0) + excluded;
|
||||
next.chunks += result.chunks;
|
||||
if (active.origin === 'manifest') next.cursor = active.next;
|
||||
next.active = null;
|
||||
});
|
||||
}
|
||||
|
||||
/** Input/provider failures stay durable; storage/durability failures must abort. */
|
||||
async fail(cause: EmbeddingFailureCause): Promise<void> {
|
||||
await this.mutate(async (next) => {
|
||||
const active = next.active;
|
||||
if (!active || !['provider', 'input', 'cancelled'].includes(cause))
|
||||
throw new Error('Invalid embedding failure');
|
||||
const record: FailedUnit = {
|
||||
version: 1,
|
||||
identity: next.identity,
|
||||
manifestDigest: next.manifestDigest,
|
||||
intent: { ...active, origin: 'failed' },
|
||||
cause,
|
||||
attempt: this.attempt,
|
||||
};
|
||||
this.uncertain = true;
|
||||
await writeEmbeddingJobRecord(this.directory, filename(active.start.unit), record);
|
||||
next.failedUnits++;
|
||||
next.failedNodes += active.nodes;
|
||||
if (active.origin === 'manifest') next.cursor = active.next;
|
||||
next.active = null;
|
||||
});
|
||||
}
|
||||
|
||||
/** Persist input-only failures before discarding partial rows; return counts for these IDs. */
|
||||
async recordInputRejections(ids: readonly string[]): Promise<Readonly<Record<string, number>>> {
|
||||
if (!ids.length) return {};
|
||||
await this.mutate(async (next) => {
|
||||
const active = next.active;
|
||||
if (!active || new Set(ids).size !== ids.length || ids.length > active.nodes)
|
||||
throw new Error('Invalid embedding input rejections');
|
||||
const history = Object.fromEntries(
|
||||
Object.entries(active.inputFailures ?? {}).map(([key, attempts]) => [
|
||||
active.inputFailureKeyFormat === 'sha256' ? key : embeddingInputFailureKey(key),
|
||||
attempts,
|
||||
]),
|
||||
);
|
||||
for (const id of ids) {
|
||||
if (!id.length || id.length > 1024 * 1024 || id === '__proto__')
|
||||
throw new Error('Invalid rejected node identity');
|
||||
const key = embeddingInputFailureKey(id);
|
||||
history[key] = Math.min(3, (history[key] ?? 0) + 1);
|
||||
}
|
||||
active.inputFailures = history;
|
||||
active.inputFailureKeyFormat = 'sha256';
|
||||
});
|
||||
const history = this.state.active!.inputFailures!;
|
||||
return Object.fromEntries(ids.map((id) => [id, history[embeddingInputFailureKey(id)]]));
|
||||
}
|
||||
|
||||
private async readFailed(name: string): Promise<FailedUnit> {
|
||||
const record = (await readEmbeddingJobRecord(this.directory, name)) as FailedUnit;
|
||||
if (
|
||||
record &&
|
||||
typeof record === 'object' &&
|
||||
((record.version !== undefined && record.version !== 1) ||
|
||||
(record.identity && record.identity !== this.state.identity) ||
|
||||
(record.manifestDigest && record.manifestDigest !== this.state.manifestDigest))
|
||||
)
|
||||
throw new Error('Embedding failure identity mismatch; recovery refused');
|
||||
if (
|
||||
!record ||
|
||||
record.version !== 1 ||
|
||||
record.identity !== this.state.identity ||
|
||||
record.manifestDigest !== this.state.manifestDigest ||
|
||||
!['provider', 'input', 'cancelled'].includes(record.cause) ||
|
||||
typeof record.attempt !== 'string'
|
||||
)
|
||||
throw new Error('Invalid failed embedding unit');
|
||||
validateIntent(record.intent, this.state);
|
||||
if (record.intent.origin !== 'failed' || name !== filename(record.intent.start.unit))
|
||||
throw new Error('Failed embedding unit identity mismatch');
|
||||
return record;
|
||||
}
|
||||
|
||||
/** Constant memory; failed records are the only growing scheduling data. */
|
||||
private async verifyFailedRecords(): Promise<void> {
|
||||
let units = 0;
|
||||
let nodes = 0;
|
||||
const entries = await fs.opendir(this.directory);
|
||||
for await (const entry of entries) {
|
||||
if (!FAILED.test(entry.name)) continue;
|
||||
const record = await this.readFailed(entry.name);
|
||||
// An active retry owns this unit even if its failure file was already
|
||||
// removed, or a crash left the previous failed record beside it.
|
||||
if (this.state.active && entry.name === filename(this.state.active.start.unit)) continue;
|
||||
units++;
|
||||
nodes += record.intent.nodes;
|
||||
if (
|
||||
record.intent.next.unit > this.state.cursor.unit ||
|
||||
units > this.state.failedUnits ||
|
||||
nodes > this.state.failedNodes
|
||||
)
|
||||
throw new Error('Invalid failed embedding accounting');
|
||||
}
|
||||
if (units !== this.state.failedUnits || nodes !== this.state.failedNodes)
|
||||
throw new Error('Invalid failed embedding accounting');
|
||||
}
|
||||
|
||||
async *failed(): AsyncGenerator<EmbeddingUnitIntent> {
|
||||
const directory = await fs.opendir(this.directory);
|
||||
for await (const entry of directory) {
|
||||
if (!FAILED.test(entry.name)) continue;
|
||||
const record = await this.readFailed(entry.name);
|
||||
if (
|
||||
record.attempt === this.attempt ||
|
||||
record.intent.start.unit === this.state.active?.start.unit
|
||||
)
|
||||
continue;
|
||||
if (record.intent.next.unit > this.state.cursor.unit)
|
||||
throw new Error('Orphan failed unit has no active intent');
|
||||
yield clone(record.intent);
|
||||
}
|
||||
}
|
||||
|
||||
async indexing(): Promise<void> {
|
||||
await this.mutate(async (next) => {
|
||||
next.phase = 'indexing';
|
||||
});
|
||||
}
|
||||
|
||||
/** The completed vector artifact must be closed and published before this call. */
|
||||
async ready(): Promise<void> {
|
||||
await this.mutate(async (next) => {
|
||||
if (next.phase !== 'indexing') throw new Error('Embedding index has not been finalized');
|
||||
next.phase = 'ready';
|
||||
});
|
||||
}
|
||||
}
|
||||
74
gitnexus/src/storage/embedding-job-progress.ts
Normal file
74
gitnexus/src/storage/embedding-job-progress.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
/** Observation only: bounded journal read, no source scan, native DB, or model. */
|
||||
import path from 'node:path';
|
||||
import {
|
||||
embeddingProgressFromState,
|
||||
type EmbeddingJobProgress,
|
||||
} from '../core/analysis-progress.js';
|
||||
import type { RepoMeta } from './repo-meta.js';
|
||||
import type { EmbeddingJobState } from './embedding-job-journal.js';
|
||||
import { embeddingJobDirectory, readEmbeddingJobRecord } from './embedding-job.js';
|
||||
import { embeddingGenerationDirectory, selectVectorGeneration } from './vector-generation.js';
|
||||
import { getExactScanLimit } from '../core/platform/capabilities.js';
|
||||
|
||||
export async function readEmbeddingJobProgress(
|
||||
metadataDirectory: string,
|
||||
metadata: RepoMeta | null | undefined,
|
||||
): Promise<EmbeddingJobProgress | undefined> {
|
||||
const reference = metadata?.embeddingJob;
|
||||
if (!reference) {
|
||||
if (
|
||||
!metadata?.vectorGeneration?.preservedForGraph &&
|
||||
!metadata?.vectorGeneration?.excludedNodes
|
||||
)
|
||||
return undefined;
|
||||
if (metadata.graphPath !== undefined && typeof metadata.graphPath !== 'string')
|
||||
return { phase: 'unverified' };
|
||||
const selected = selectVectorGeneration(
|
||||
metadataDirectory,
|
||||
metadata,
|
||||
path.resolve(metadataDirectory, metadata.graphPath ?? 'lbug'),
|
||||
getExactScanLimit(),
|
||||
);
|
||||
// These are historical chunks, not completed/reused work in a current
|
||||
// manifest. Unknown selection totals stay absent, never zero or complete.
|
||||
return selected.kind === 'ready'
|
||||
? {
|
||||
phase: selected.coverage === 'preserved' ? 'preserved' : 'ready',
|
||||
chunks: selected.generation.chunks,
|
||||
...(selected.generation.excludedNodes
|
||||
? { excludedNodes: selected.generation.excludedNodes }
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
try {
|
||||
if (
|
||||
reference.version !== 1 ||
|
||||
metadata?.graphGeneration?.version !== 1 ||
|
||||
!/^[a-f0-9]{32}$/.test(reference.graphGeneration) ||
|
||||
reference.graphGeneration !== metadata.graphGeneration.id
|
||||
)
|
||||
throw new Error('Job graph mismatch');
|
||||
const directory = embeddingGenerationDirectory(metadataDirectory, reference.identity);
|
||||
// Validate every job-owned directory before following the path. In
|
||||
// particular, a symlinked embeddings parent must not redirect this read.
|
||||
await embeddingJobDirectory(metadataDirectory);
|
||||
await embeddingJobDirectory(path.dirname(directory));
|
||||
const state = (await readEmbeddingJobRecord(directory, 'job.json')) as EmbeddingJobState;
|
||||
if (
|
||||
!state ||
|
||||
state.version !== 1 ||
|
||||
state.identity !== reference.identity ||
|
||||
!/^[a-f0-9]{64}$/.test(state.manifestDigest)
|
||||
)
|
||||
throw new Error('Job identity mismatch');
|
||||
const progress = embeddingProgressFromState(state);
|
||||
// A journal alone cannot publish a vector capability. If the metadata
|
||||
// still points at pending work, even a ready journal awaits publication.
|
||||
if (progress.phase === 'ready') progress.phase = 'indexing';
|
||||
return progress;
|
||||
} catch {
|
||||
// Missing/corrupt/unreadable work is not a zero-work or completed job.
|
||||
return { phase: 'unverified' };
|
||||
}
|
||||
}
|
||||
164
gitnexus/src/storage/embedding-job.ts
Normal file
164
gitnexus/src/storage/embedding-job.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
/** Filesystem-only embedding receipts; importing this never loads a model or DB. */
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { constants, type Stats } from 'node:fs';
|
||||
import fs, { type FileHandle } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const MAX_RECORD_BYTES = 32 * 1024;
|
||||
const RECORD_NAMES = new Set([
|
||||
'manifest.json',
|
||||
'job.json',
|
||||
'space.json',
|
||||
'rebase.json',
|
||||
'legacy.json',
|
||||
'retained.json',
|
||||
'artifact.json',
|
||||
]);
|
||||
const FAILED_RECORD_NAME = /^failed-\d{16}\.json$/;
|
||||
|
||||
/** Pin the file before checking it; a replaced pathname cannot redirect reads. */
|
||||
export async function openEmbeddingJobFile(
|
||||
file: string,
|
||||
): Promise<{ handle: FileHandle; stat: Stats }> {
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
file,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ELOOP')
|
||||
throw new Error('Embedding artifact must be a regular file', { cause: error });
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) throw new Error('Embedding artifact must be a regular file');
|
||||
// Windows lacks O_NOFOLLOW. Validate the directory entry against the
|
||||
// pinned descriptor before reading any bytes; later path swaps are harmless.
|
||||
if (!constants.O_NOFOLLOW) {
|
||||
const entry = await fs.lstat(file);
|
||||
if (!entry.isFile() || entry.isSymbolicLink())
|
||||
throw new Error('Embedding artifact must be a regular file');
|
||||
if (entry.dev !== stat.dev || entry.ino !== stat.ino)
|
||||
throw new Error('Embedding artifact changed during open');
|
||||
}
|
||||
return { handle, stat };
|
||||
} catch (error) {
|
||||
await handle.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function embeddingJobDirectory(directory: string): Promise<string> {
|
||||
const stat = await fs.lstat(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink())
|
||||
throw new Error('Embedding job directory must be an owned directory');
|
||||
const canonical = await fs.realpath(directory);
|
||||
const resolved = await fs.lstat(canonical);
|
||||
if (
|
||||
!resolved.isDirectory() ||
|
||||
resolved.isSymbolicLink() ||
|
||||
resolved.dev !== stat.dev ||
|
||||
resolved.ino !== stat.ino
|
||||
)
|
||||
throw new Error('Embedding job directory changed during resolution');
|
||||
return canonical;
|
||||
}
|
||||
|
||||
/** Root is already canonical and validated; validate each child before traversing it. */
|
||||
export async function createEmbeddingJobDirectories(
|
||||
root: string,
|
||||
components: readonly string[],
|
||||
): Promise<string> {
|
||||
let directory = root;
|
||||
for (const component of components) {
|
||||
const next = path.join(directory, component);
|
||||
try {
|
||||
await fs.mkdir(next, { mode: 0o700 });
|
||||
await syncEmbeddingJobDirectory(directory);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
||||
}
|
||||
directory = await embeddingJobDirectory(next);
|
||||
}
|
||||
return directory;
|
||||
}
|
||||
|
||||
/** Flush directory entries where the platform supports opening directories. */
|
||||
export async function syncEmbeddingJobDirectory(directory: string): Promise<void> {
|
||||
if (process.platform === 'win32') return;
|
||||
const handle = await fs.open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0));
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeEmbeddingJobRecord(
|
||||
directory: string,
|
||||
name: string,
|
||||
payload: unknown,
|
||||
): Promise<void> {
|
||||
if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name))
|
||||
throw new Error('Invalid embedding receipt name');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
const serialized = JSON.stringify(payload);
|
||||
if (serialized === undefined) throw new Error('Embedding receipt payload is not JSON');
|
||||
const checksum = createHash('sha256').update(serialized).digest('hex');
|
||||
const bytes = Buffer.from(JSON.stringify({ checksum, payload }), 'utf8');
|
||||
if (bytes.byteLength > MAX_RECORD_BYTES)
|
||||
throw new Error('Embedding receipt exceeds its fixed size limit');
|
||||
const temporary = path.join(canonical, `.${name}.${randomUUID()}.tmp`);
|
||||
const handle = await fs.open(temporary, 'wx', 0o600);
|
||||
try {
|
||||
try {
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await fs.rename(temporary, path.join(canonical, name));
|
||||
await syncEmbeddingJobDirectory(canonical);
|
||||
} finally {
|
||||
await fs.rm(temporary, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Bounded read; malformed or untrusted receipts never become an empty job. */
|
||||
export async function readEmbeddingJobRecord(directory: string, name: string): Promise<unknown> {
|
||||
if (!RECORD_NAMES.has(name) && !FAILED_RECORD_NAME.test(name))
|
||||
throw new Error('Invalid embedding receipt name');
|
||||
const canonical = await embeddingJobDirectory(directory);
|
||||
const target = path.join(canonical, name);
|
||||
const { handle } = await openEmbeddingJobFile(target);
|
||||
try {
|
||||
const buffer = Buffer.alloc(MAX_RECORD_BYTES + 1);
|
||||
let length = 0;
|
||||
while (length < buffer.length) {
|
||||
const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length);
|
||||
if (bytesRead === 0) break;
|
||||
length += bytesRead;
|
||||
}
|
||||
if (length > MAX_RECORD_BYTES)
|
||||
throw new Error('Embedding receipt exceeds its fixed size limit');
|
||||
const parsed: unknown = JSON.parse(
|
||||
new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)),
|
||||
);
|
||||
if (
|
||||
!parsed ||
|
||||
typeof parsed !== 'object' ||
|
||||
!('checksum' in parsed) ||
|
||||
!('payload' in parsed)
|
||||
) {
|
||||
throw new Error('Malformed embedding receipt');
|
||||
}
|
||||
const checksum = createHash('sha256').update(JSON.stringify(parsed.payload)).digest('hex');
|
||||
if (parsed.checksum !== checksum)
|
||||
throw new Error('Embedding receipt checksum mismatch; artifacts retained');
|
||||
return parsed.payload;
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { execFileSync, execSync, spawnSync } from 'child_process';
|
||||
import { statSync, existsSync } from 'fs';
|
||||
import { statSync, existsSync, lstatSync, realpathSync } from 'fs';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
|
@ -585,6 +585,21 @@ export const getCoreExcludesFilePath = (fromPath: string): string => {
|
|||
*/
|
||||
export const resolveRepoIdentityRoot = (fromPath: string): string => {
|
||||
const resolved = path.resolve(fromPath);
|
||||
// An ordinary checkout is its own identity root. Let Git interpret worktrees,
|
||||
// symlinks, common-directory indirection and environment overrides as before.
|
||||
try {
|
||||
const gitDir = path.join(resolved, '.git');
|
||||
if (
|
||||
process.env.GIT_DIR === undefined &&
|
||||
process.env.GIT_COMMON_DIR === undefined &&
|
||||
lstatSync(gitDir).isDirectory() &&
|
||||
!existsSync(path.join(gitDir, 'commondir')) &&
|
||||
realpathSync.native(resolved) === resolved
|
||||
)
|
||||
return resolved;
|
||||
} catch {
|
||||
// Missing or unreadable paths retain the existing Git/fallback behavior.
|
||||
}
|
||||
const canonical = getCanonicalRepoRoot(resolved);
|
||||
if (!canonical) return resolved; // non-git → use as-is
|
||||
if (canonical === resolved) return canonical; // canonical checkout
|
||||
|
|
|
|||
66
gitnexus/src/storage/index-capabilities.ts
Normal file
66
gitnexus/src/storage/index-capabilities.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
/** Published capability receipts, gated by current routing and artifact ownership.
|
||||
* Runtime query probes may further degrade these facts; status never loads a DB. */
|
||||
import type { RepoMeta } from './repo-meta.js';
|
||||
import { resolveGraphPath } from './shared-store.js';
|
||||
import { assertOwnedVectorGeneration, selectVectorGeneration } from './vector-generation.js';
|
||||
import { getExactScanLimit } from '../core/platform/capabilities.js';
|
||||
|
||||
export async function readIndexCapabilities(
|
||||
directory: string,
|
||||
metadata: RepoMeta | null | undefined,
|
||||
): Promise<RepoMeta['capabilities']> {
|
||||
if (
|
||||
!metadata?.graphGeneration &&
|
||||
!metadata?.vectorGeneration &&
|
||||
!metadata?.embeddingJob &&
|
||||
!metadata?.incrementalInProgress &&
|
||||
!metadata?.graphWriteCollapsed
|
||||
)
|
||||
return metadata?.capabilities;
|
||||
const exactScanLimit = getExactScanLimit();
|
||||
let vectorSearch: NonNullable<RepoMeta['capabilities']>['vectorSearch'];
|
||||
try {
|
||||
const selection = selectVectorGeneration(
|
||||
directory,
|
||||
metadata,
|
||||
resolveGraphPath(directory, metadata),
|
||||
exactScanLimit,
|
||||
);
|
||||
if (selection.kind === 'ready') {
|
||||
await assertOwnedVectorGeneration(
|
||||
directory,
|
||||
selection.generation.artifactIdentity ?? selection.generation.identity,
|
||||
);
|
||||
vectorSearch = {
|
||||
provider: selection.generation.indexKind === 'hnsw' ? 'ladybugdb-vector' : 'exact-scan',
|
||||
status: selection.generation.indexKind === 'hnsw' ? 'vector-index' : 'exact-scan',
|
||||
exactScanLimit,
|
||||
coverage: selection.coverage ?? 'current',
|
||||
sourceGraphGeneration: selection.generation.graphGeneration,
|
||||
...(selection.generation.excludedNodes ? { reason: 'embedding-inputs-rejected' } : {}),
|
||||
};
|
||||
} else if (selection.kind === 'legacy' && metadata?.capabilities?.vectorSearch) {
|
||||
vectorSearch = { ...metadata.capabilities.vectorSearch, coverage: 'preserved' };
|
||||
} else {
|
||||
vectorSearch = {
|
||||
provider: 'ladybugdb-vector',
|
||||
status: 'unavailable',
|
||||
exactScanLimit,
|
||||
reason:
|
||||
selection.kind === 'unavailable' ? selection.reason : 'vector-generation-unverified',
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
vectorSearch = {
|
||||
provider: 'ladybugdb-vector',
|
||||
status: 'unavailable',
|
||||
exactScanLimit,
|
||||
reason: 'vector-artifact-unavailable',
|
||||
};
|
||||
}
|
||||
return {
|
||||
graph: metadata?.capabilities?.graph ?? { provider: 'ladybugdb', status: 'unavailable' },
|
||||
fts: metadata?.capabilities?.fts ?? { provider: 'ladybugdb-fts', status: 'unavailable' },
|
||||
vectorSearch,
|
||||
};
|
||||
}
|
||||
|
|
@ -226,7 +226,7 @@ const unverifiedGuardError = (guardPath: string): Error =>
|
|||
const HOSTNAME = os.hostname();
|
||||
|
||||
/** Linux: field 22 of /proc/<pid>/stat (starttime). null elsewhere / on error. */
|
||||
const readProcStartTime = (pid: number): string | null => {
|
||||
export const readProcStartTime = (pid: number): string | null => {
|
||||
if (process.platform !== 'linux') return null;
|
||||
try {
|
||||
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
||||
|
|
@ -441,9 +441,19 @@ const deniedCreateHandle = (
|
|||
* `branches/<slug>/` sub-slot (which owns its own lock + sweep). Non-recursive.
|
||||
*/
|
||||
export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => void): void => {
|
||||
// A graph worker releases this lock before its supervisor reacquires it.
|
||||
// Presence (even malformed or unreadable) retains the attempt until the
|
||||
// publication coordinator validates it; lock ownership alone cannot prove
|
||||
// that these stages are orphaned.
|
||||
try {
|
||||
lstatSync(path.join(lockDir, 'analysis-publication.json'));
|
||||
return;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return;
|
||||
}
|
||||
// Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.<id>`, `lbug.staging.<id>.wal`, …
|
||||
// Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`.
|
||||
const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/;
|
||||
const stagingRe = /^lbug\.(?:generation\.[a-f0-9]{32}\.)?(staging\..+|new(\..+)?)$/;
|
||||
const retained = new Set(readEmbeddingRecovery(lockDir)?.familyFiles ?? []);
|
||||
let removed = 0;
|
||||
let entries: string[];
|
||||
|
|
|
|||
295
gitnexus/src/storage/legacy-embedding-source.ts
Normal file
295
gitnexus/src/storage/legacy-embedding-source.ts
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
/** Filesystem-only provenance for one retired, immutable legacy database.
|
||||
* The caller holds the slot lock and retires/checkpoints the source before
|
||||
* recording it. No current configuration is used to fill historical fields. */
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { BigIntStats } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { EmbeddingIdentity } from '../core/embeddings/embedding-identity.js';
|
||||
import { assertValidGraphNodeId } from '../core/graph/node-identity.js';
|
||||
import {
|
||||
embeddingJobDirectory,
|
||||
openEmbeddingJobFile,
|
||||
readEmbeddingJobRecord,
|
||||
writeEmbeddingJobRecord,
|
||||
syncEmbeddingJobDirectory,
|
||||
} from './embedding-job.js';
|
||||
|
||||
const DIGEST = /^[a-f0-9]{64}$/;
|
||||
const SOURCE =
|
||||
/^lbug\.generation\.[a-f0-9]{32}$|^lbug(?:\.generation\.[a-f0-9]{32})?\.staging\.[a-f0-9-]{36}$/;
|
||||
const IO_BYTES = 64 * 1024;
|
||||
const MAX_ID_BYTES = 1024 * 1024;
|
||||
const hash = (value: unknown): string =>
|
||||
createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
const count = (value: unknown): value is number =>
|
||||
typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
||||
|
||||
export interface LegacyEmbeddingSourceRecord {
|
||||
version: 1;
|
||||
sourceFile: string;
|
||||
schemaFingerprint: string;
|
||||
recordedIdentity?: Partial<EmbeddingIdentity>;
|
||||
recordedInputIdentity?: string;
|
||||
/** Descriptor attributes detect replacement/mutation. Ownership and provider
|
||||
* provenance come from the retired slot and record, never these attributes. */
|
||||
file: { dev: string; ino: string; size: string; mtimeNs: string; ctimeNs: string };
|
||||
excluded: { digest: string; bytes: number; nodes: number };
|
||||
}
|
||||
export interface ResolvedLegacyEmbeddingSource extends LegacyEmbeddingSourceRecord {
|
||||
identity: string;
|
||||
path: string;
|
||||
excludedIds: () => AsyncIterable<string>;
|
||||
}
|
||||
|
||||
function identityFields(
|
||||
value: Partial<EmbeddingIdentity> | undefined,
|
||||
): Partial<EmbeddingIdentity> | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
(value.provider !== undefined && !/^(local|http:[a-f0-9]{64})$/.test(value.provider)) ||
|
||||
(value.model !== undefined &&
|
||||
(typeof value.model !== 'string' || !value.model || value.model.length > 2048)) ||
|
||||
(value.dimensions !== undefined && (!count(value.dimensions) || !value.dimensions))
|
||||
)
|
||||
throw new Error('Invalid recorded legacy embedding identity');
|
||||
return {
|
||||
...(value.provider === undefined ? {} : { provider: value.provider }),
|
||||
...(value.model === undefined ? {} : { model: value.model }),
|
||||
...(value.dimensions === undefined ? {} : { dimensions: value.dimensions }),
|
||||
};
|
||||
}
|
||||
function record(value: unknown): LegacyEmbeddingSourceRecord {
|
||||
const r = value as LegacyEmbeddingSourceRecord;
|
||||
if (
|
||||
!r ||
|
||||
r.version !== 1 ||
|
||||
typeof r.sourceFile !== 'string' ||
|
||||
!SOURCE.test(r.sourceFile) ||
|
||||
typeof r.schemaFingerprint !== 'string' ||
|
||||
!r.schemaFingerprint ||
|
||||
r.schemaFingerprint.length > 2048 ||
|
||||
(r.recordedInputIdentity !== undefined && !DIGEST.test(r.recordedInputIdentity)) ||
|
||||
!r.file ||
|
||||
![r.file.dev, r.file.ino, r.file.size, r.file.mtimeNs, r.file.ctimeNs].every(
|
||||
(v) => typeof v === 'string' && /^\d+$/.test(v),
|
||||
) ||
|
||||
!r.excluded ||
|
||||
!DIGEST.test(r.excluded.digest) ||
|
||||
!count(r.excluded.bytes) ||
|
||||
!count(r.excluded.nodes)
|
||||
)
|
||||
throw new Error('Invalid legacy source receipt; source retained');
|
||||
const recordedIdentity = identityFields(r.recordedIdentity);
|
||||
return {
|
||||
version: 1,
|
||||
sourceFile: r.sourceFile,
|
||||
schemaFingerprint: r.schemaFingerprint,
|
||||
...(recordedIdentity === undefined ? {} : { recordedIdentity }),
|
||||
...(r.recordedInputIdentity === undefined
|
||||
? {}
|
||||
: { recordedInputIdentity: r.recordedInputIdentity }),
|
||||
file: {
|
||||
dev: r.file.dev,
|
||||
ino: r.file.ino,
|
||||
size: r.file.size,
|
||||
mtimeNs: r.file.mtimeNs,
|
||||
ctimeNs: r.file.ctimeNs,
|
||||
},
|
||||
excluded: { digest: r.excluded.digest, bytes: r.excluded.bytes, nodes: r.excluded.nodes },
|
||||
};
|
||||
}
|
||||
function fileIdentity(stat: BigIntStats): LegacyEmbeddingSourceRecord['file'] {
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Legacy source must be an owned regular file');
|
||||
return {
|
||||
dev: String(stat.dev),
|
||||
ino: String(stat.ino),
|
||||
size: String(stat.size),
|
||||
mtimeNs: String(stat.mtimeNs),
|
||||
ctimeNs: String(stat.ctimeNs),
|
||||
};
|
||||
}
|
||||
async function sourceFile(root: string, r: LegacyEmbeddingSourceRecord): Promise<string> {
|
||||
const file = path.join(root, r.sourceFile);
|
||||
if (
|
||||
JSON.stringify(fileIdentity(await fs.lstat(file, { bigint: true }))) !== JSON.stringify(r.file)
|
||||
)
|
||||
throw new Error('Legacy source changed after retirement; artifacts retained');
|
||||
// A read-only migration cannot recover an interrupted native write. Its
|
||||
// graph-phase owner must checkpoint/recover it before writing this receipt.
|
||||
for (const suffix of ['.wal', '.shadow']) {
|
||||
try {
|
||||
const stat = await fs.lstat(file + suffix);
|
||||
if (!stat.isFile() || stat.size)
|
||||
throw new Error('Legacy source needs native recovery before reuse');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
return file;
|
||||
}
|
||||
async function legacyRoot(root: string, create: boolean): Promise<string> {
|
||||
let directory = await embeddingJobDirectory(root);
|
||||
for (const part of ['embeddings', 'legacy']) {
|
||||
const next = path.join(directory, part);
|
||||
if (create) {
|
||||
try {
|
||||
await fs.mkdir(next, { mode: 0o700 });
|
||||
await syncEmbeddingJobDirectory(directory);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
||||
}
|
||||
}
|
||||
directory = await embeddingJobDirectory(next);
|
||||
}
|
||||
return directory;
|
||||
}
|
||||
|
||||
/** Streams checkpoint exclusions once. The small receipt is published last,
|
||||
* after the spool is durable. The native source is never copied or modified. */
|
||||
export async function recordLegacyEmbeddingSource(
|
||||
root: string,
|
||||
input: Pick<
|
||||
LegacyEmbeddingSourceRecord,
|
||||
'sourceFile' | 'schemaFingerprint' | 'recordedIdentity' | 'recordedInputIdentity'
|
||||
>,
|
||||
excluded: AsyncIterable<string> | Iterable<string>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
root = await embeddingJobDirectory(root);
|
||||
if (!SOURCE.test(input.sourceFile))
|
||||
throw new Error('Legacy source must be a retired owned graph');
|
||||
const file = fileIdentity(await fs.lstat(path.join(root, input.sourceFile), { bigint: true }));
|
||||
const parent = await legacyRoot(root, true);
|
||||
const temporary = path.join(parent, `.prepare-${randomUUID()}`);
|
||||
await fs.mkdir(temporary, { mode: 0o700 });
|
||||
try {
|
||||
const handle = await fs.open(path.join(temporary, 'excluded.jsonl'), 'wx', 0o600);
|
||||
const checksum = createHash('sha256');
|
||||
let bytes = 0,
|
||||
nodes = 0,
|
||||
pending = '',
|
||||
pendingBytes = 0;
|
||||
try {
|
||||
for await (const id of excluded) {
|
||||
signal?.throwIfAborted();
|
||||
assertValidGraphNodeId(id, 'Legacy checkpoint');
|
||||
const line = JSON.stringify(id) + '\n';
|
||||
const length = Buffer.byteLength(line);
|
||||
if (length > MAX_ID_BYTES || !Number.isSafeInteger(bytes + length))
|
||||
throw new Error('Legacy exclusion exceeds its bounded size');
|
||||
checksum.update(line);
|
||||
bytes += length;
|
||||
nodes++;
|
||||
pending += line;
|
||||
pendingBytes += length;
|
||||
if (pendingBytes >= IO_BYTES) {
|
||||
await handle.writeFile(pending);
|
||||
pending = '';
|
||||
pendingBytes = 0;
|
||||
}
|
||||
}
|
||||
if (pendingBytes) await handle.writeFile(pending);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
const receipt = record({
|
||||
version: 1,
|
||||
...input,
|
||||
file,
|
||||
excluded: { digest: checksum.digest('hex'), bytes, nodes },
|
||||
});
|
||||
await sourceFile(root, receipt);
|
||||
const identity = hash(receipt);
|
||||
await writeEmbeddingJobRecord(temporary, 'legacy.json', receipt);
|
||||
const destination = path.join(parent, identity);
|
||||
try {
|
||||
await fs.rename(temporary, destination);
|
||||
} catch (error) {
|
||||
if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? ''))
|
||||
throw error;
|
||||
// An identical request is idempotent, but a corrupt prior spool is not
|
||||
// silently replaced. Fully validate it before trusting the same name.
|
||||
const prior = await readLegacyEmbeddingSource(root, identity);
|
||||
for await (const _id of prior.excludedIds()) signal?.throwIfAborted();
|
||||
}
|
||||
await syncEmbeddingJobDirectory(parent);
|
||||
return identity;
|
||||
} finally {
|
||||
await fs.rm(temporary, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function* exclusions(
|
||||
directory: string,
|
||||
expected: LegacyEmbeddingSourceRecord['excluded'],
|
||||
): AsyncGenerator<string> {
|
||||
const file = path.join(directory, 'excluded.jsonl');
|
||||
const { handle, stat } = await openEmbeddingJobFile(file);
|
||||
try {
|
||||
if (stat.size !== expected.bytes) throw new Error('Legacy exclusions size mismatch');
|
||||
const buffer = Buffer.alloc(IO_BYTES),
|
||||
checksum = createHash('sha256');
|
||||
let pending = Buffer.alloc(0),
|
||||
bytes = 0,
|
||||
nodes = 0;
|
||||
for (;;) {
|
||||
const { bytesRead } = await handle.read(buffer, 0, buffer.length, bytes);
|
||||
if (!bytesRead) break;
|
||||
bytes += bytesRead;
|
||||
checksum.update(buffer.subarray(0, bytesRead));
|
||||
const data = pending.length
|
||||
? Buffer.concat([pending, buffer.subarray(0, bytesRead)])
|
||||
: buffer.subarray(0, bytesRead);
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const end = data.indexOf(10, offset);
|
||||
if (end < 0) break;
|
||||
if (end - offset + 1 > MAX_ID_BYTES)
|
||||
throw new Error('Legacy exclusion exceeds its bounded size');
|
||||
const id: unknown = JSON.parse(
|
||||
new TextDecoder('utf-8', { fatal: true }).decode(data.subarray(offset, end)),
|
||||
);
|
||||
assertValidGraphNodeId(id, 'Legacy checkpoint');
|
||||
nodes++;
|
||||
yield id;
|
||||
offset = end + 1;
|
||||
}
|
||||
pending = Buffer.from(data.subarray(offset));
|
||||
if (pending.length >= MAX_ID_BYTES || bytes > expected.bytes)
|
||||
throw new Error('Legacy exclusions exceed their bounds');
|
||||
}
|
||||
if (
|
||||
pending.length ||
|
||||
bytes !== expected.bytes ||
|
||||
nodes !== expected.nodes ||
|
||||
checksum.digest('hex') !== expected.digest
|
||||
)
|
||||
throw new Error('Legacy exclusions checksum mismatch; source retained');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readLegacyEmbeddingSource(
|
||||
root: string,
|
||||
identity: string,
|
||||
): Promise<ResolvedLegacyEmbeddingSource> {
|
||||
if (!DIGEST.test(identity)) throw new Error('Invalid legacy source identity');
|
||||
root = await embeddingJobDirectory(root);
|
||||
const directory = await embeddingJobDirectory(path.join(await legacyRoot(root, false), identity));
|
||||
const receipt = record(await readEmbeddingJobRecord(directory, 'legacy.json'));
|
||||
if (hash(receipt) !== identity)
|
||||
throw new Error('Legacy source identity mismatch; source retained');
|
||||
return {
|
||||
...receipt,
|
||||
identity,
|
||||
path: await sourceFile(root, receipt),
|
||||
excludedIds: () => exclusions(directory, receipt.excluded),
|
||||
};
|
||||
}
|
||||
|
|
@ -50,7 +50,6 @@ import {
|
|||
type AnalyzerRunnerIdentity,
|
||||
type RepoMeta,
|
||||
} from './repo-meta.js';
|
||||
import { LBUG_DIRECTORY } from './storage-constants.js';
|
||||
import { resolveGraphPath } from './shared-store.js';
|
||||
import {
|
||||
defaultStoragePath,
|
||||
|
|
@ -243,9 +242,9 @@ export const getStoragePaths = (
|
|||
const baseDir = branch ? path.join(storagePath, BRANCHES_DIR, branchSlug(branch)) : storagePath;
|
||||
return {
|
||||
storagePath,
|
||||
// Branch slots are always private; a flat shared-store slot may read a
|
||||
// commit graph (#3352).
|
||||
lbugPath: branch ? path.join(baseDir, LBUG_DIRECTORY) : resolveGraphPath(storagePath),
|
||||
// Private generations also work in branch slots. Only a flat shared-store
|
||||
// slot may resolve to an immutable commit graph (#3352).
|
||||
lbugPath: resolveGraphPath(baseDir),
|
||||
metaPath: path.join(baseDir, INDEX_METADATA_FILE), // Branch-specific metadata file
|
||||
};
|
||||
};
|
||||
|
|
@ -945,6 +944,7 @@ export class RegistryNameCollisionError extends Error {
|
|||
const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): boolean => {
|
||||
const resolved = path.resolve(entry.path);
|
||||
if (entry.name === path.basename(resolved)) return false;
|
||||
if (inferredName && entry.name === inferredName) return false;
|
||||
// Canonical-root-derived names are not user aliases either (#1259):
|
||||
// a worktree registered under the canonical repo's basename
|
||||
// (e.g. `{name: 'repo', path: '/repo/wt-feature'}`) must re-register
|
||||
|
|
@ -956,7 +956,6 @@ const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): bool
|
|||
// — analyze canonical, then analyze worktree, then re-analyze worktree
|
||||
// — would break on the third call.
|
||||
if (entry.name === path.basename(resolveRepoIdentityRoot(resolved))) return false;
|
||||
if (inferredName && entry.name === inferredName) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -114,11 +114,51 @@ export interface RepoMeta {
|
|||
/** Complete index directory selected for this successful analysis. */
|
||||
storagePath?: string;
|
||||
/**
|
||||
* Shared-store checkouts only (#3352): the graph this slot reads when it is
|
||||
* not `<storagePath>/lbug` — a commit graph under the same store. Resolved
|
||||
* and validated by `resolveGraphPath`.
|
||||
* The graph this slot reads: an owned private generation basename, or an
|
||||
* absolute shared commit graph under the same store (#3352). Resolved and
|
||||
* validated by `resolveGraphPath`. Absent means legacy `<storagePath>/lbug`.
|
||||
*/
|
||||
graphPath?: string;
|
||||
/** A verified logical generation, published independently of optional vectors.
|
||||
* Private incremental writes invalidate this receipt before mutating its file. */
|
||||
graphGeneration?: {
|
||||
version: 1;
|
||||
id: string;
|
||||
/** Digest of covered content, including uncommitted source changes. */
|
||||
coveredContent: string;
|
||||
};
|
||||
/** Checkout-owned durable work. Progress lives in its bounded journal. */
|
||||
embeddingJob?: {
|
||||
version: 1;
|
||||
identity: string;
|
||||
graphGeneration: string;
|
||||
/** Mutable artifact ownership is separate from this graph-bound job. */
|
||||
artifactIdentity?: string;
|
||||
spaceKey?: string;
|
||||
};
|
||||
/** Direct receipt for a retained legacy source; missing historical fields
|
||||
* remain unknown. The source is immutable and separate from graphPath. */
|
||||
legacyVectorSource?: string;
|
||||
/** Published only after complete coverage, native durability and index validation. */
|
||||
vectorGeneration?: {
|
||||
version: 1;
|
||||
identity: string;
|
||||
/** Absent in older separate stores, whose artifact identity equaled the job. */
|
||||
artifactIdentity?: string;
|
||||
graphGeneration: string;
|
||||
/** Explicit permission to serve this completed historical artifact against
|
||||
* a newer graph. graphGeneration remains its original coverage receipt. */
|
||||
preservedForGraph?: string;
|
||||
vectorIdentity: string;
|
||||
inputIdentity: string;
|
||||
dimensions: number;
|
||||
nodes: number;
|
||||
chunks: number;
|
||||
/** Deterministic input failures exhausted three attempts in this job. */
|
||||
excludedNodes?: number;
|
||||
exclusionReason?: 'input-rejected';
|
||||
indexKind: 'hnsw' | 'exact';
|
||||
};
|
||||
lastCommit: string;
|
||||
indexedAt: string;
|
||||
/** Missing on legacy metadata means the upstream-compatible `full` profile. */
|
||||
|
|
@ -248,6 +288,9 @@ export interface RepoMeta {
|
|||
status: 'vector-index' | 'exact-scan' | 'unavailable';
|
||||
exactScanLimit: number;
|
||||
reason?: string;
|
||||
/** Native availability and coverage of the current selection differ. */
|
||||
coverage?: 'current' | 'preserved' | 'unknown';
|
||||
sourceGraphGeneration?: string;
|
||||
};
|
||||
};
|
||||
/**
|
||||
|
|
@ -459,6 +502,8 @@ export interface RepoMeta {
|
|||
* to a known-good index.
|
||||
*/
|
||||
incrementalInProgress?: {
|
||||
/** Unique ownership fence for a prepared in-place graph publication. */
|
||||
attemptId?: string;
|
||||
/** When the run started (epoch ms). */
|
||||
startedAt: number;
|
||||
/** Last dirty-flag refresh (epoch ms). */
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import { existsSync } from 'fs';
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { acquireIndexLock, requireExclusiveIndexLock, type IndexLockHandle } from './index-lock.js';
|
||||
import { sharedMemberGatePath } from './analysis-slot-lock.js';
|
||||
import {
|
||||
canonicalizePath,
|
||||
findRegistryEntryByRepoPath,
|
||||
|
|
@ -25,12 +26,7 @@ import {
|
|||
storeRootOfCheckoutSlot,
|
||||
type SharedStoreLayout,
|
||||
} from './shared-store.js';
|
||||
import {
|
||||
GITNEXUS_DIR,
|
||||
INDEX_METADATA_FILE,
|
||||
LBUG_DIRECTORY,
|
||||
LEGACY_METADATA_FILE,
|
||||
} from './storage-constants.js';
|
||||
import { GITNEXUS_DIR, INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './storage-constants.js';
|
||||
|
||||
type StoreRoot = Pick<SharedStoreLayout, 'root'>;
|
||||
|
||||
|
|
@ -105,6 +101,50 @@ const loadMetaStrict = async (slot: string): Promise<RepoMeta | null> => {
|
|||
return null;
|
||||
};
|
||||
|
||||
/** Each branch owns its own publication and vector work under the member's
|
||||
* ordinary slot lock. Never follow a symlink while deciding what to collect. */
|
||||
async function memberMetadataDirectories(slot: string): Promise<string[]> {
|
||||
const directories = [slot];
|
||||
const branches = path.join(slot, 'branches');
|
||||
try {
|
||||
const stat = await fs.lstat(branches);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink())
|
||||
throw new Error('Cannot collect a shared member with an unverified branches directory');
|
||||
for (const entry of await fs.readdir(branches, { withFileTypes: true })) {
|
||||
if (entry.isSymbolicLink())
|
||||
throw new Error('Cannot collect a shared member with a symlinked branch');
|
||||
if (entry.isDirectory() && !entry.name.startsWith('.'))
|
||||
directories.push(path.join(branches, entry.name));
|
||||
}
|
||||
} catch (error) {
|
||||
if (!isMissingFilesystemError(error)) throw error;
|
||||
}
|
||||
return directories;
|
||||
}
|
||||
|
||||
/** The writer releases its lock before an exit-only supervisor reacquires it.
|
||||
* Its durable intent still owns the files throughout that gap and a restart.
|
||||
* A malformed receipt is retained evidence, never permission to delete it. */
|
||||
async function hasRetainedAnalysis(slot: string): Promise<boolean> {
|
||||
for (const directory of await memberMetadataDirectories(slot)) {
|
||||
try {
|
||||
await fs.lstat(path.join(directory, 'analysis-publication.json'));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (!isMissingFilesystemError(error)) throw error;
|
||||
}
|
||||
const metadata = await loadMetaStrict(directory);
|
||||
if (
|
||||
metadata &&
|
||||
(Object.hasOwn(metadata, 'embeddingJob') ||
|
||||
Object.hasOwn(metadata, 'embeddingCheckpoint') ||
|
||||
Object.hasOwn(metadata, 'legacyVectorSource'))
|
||||
)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Member slots that no registry entry uses any more: the checkout directory is
|
||||
* gone, or its entry moved elsewhere (`--no-share`, sharing turned off). The
|
||||
|
|
@ -121,7 +161,7 @@ const orphanMembers = async (slots: string[]): Promise<Set<string>> => {
|
|||
const meta = await loadMeta(slot);
|
||||
if (!meta?.repoPath) continue;
|
||||
if (!existsSync(meta.repoPath)) {
|
||||
orphans.add(slot);
|
||||
if (!(await hasRetainedAnalysis(slot))) orphans.add(slot);
|
||||
continue;
|
||||
}
|
||||
if (!entries) continue;
|
||||
|
|
@ -130,7 +170,7 @@ const orphanMembers = async (slots: string[]): Promise<Set<string>> => {
|
|||
!entry ||
|
||||
!registryPathEquals(canonicalizePath(entry.storagePath), canonicalizePath(slot))
|
||||
) {
|
||||
orphans.add(slot);
|
||||
if (!(await hasRetainedAnalysis(slot))) orphans.add(slot);
|
||||
}
|
||||
}
|
||||
return orphans;
|
||||
|
|
@ -158,87 +198,128 @@ export const reclaimSharedStoreLocked = async (
|
|||
const checkoutsDir = path.join(storeRoot, 'checkouts');
|
||||
const commitsDir = path.join(storeRoot, 'commits');
|
||||
const referenced = new Set<string>();
|
||||
let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name));
|
||||
if (opts.gc) {
|
||||
const orphans = await orphanMembers(slots);
|
||||
for (const slot of [...orphans]) {
|
||||
// An analyze holds its slot's index lock until it has registered the
|
||||
// checkout, so a slot that is seeded but not yet registered is busy,
|
||||
// not orphaned. Judge (and, unless previewing, delete) only a slot
|
||||
// whose lock is free, so the preview matches what --force would do.
|
||||
let lock: IndexLockHandle;
|
||||
try {
|
||||
// A preview deletes nothing, not even the staging files the lock sweeps.
|
||||
lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: !opts.dryRun });
|
||||
} catch {
|
||||
orphans.delete(slot);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (lock.lockFree || !(await orphanMembers([slot])).has(slot)) {
|
||||
const heldMemberGates: IndexLockHandle[] = [];
|
||||
try {
|
||||
let slots = (await listDirStrict(checkoutsDir)).map((name) => path.join(checkoutsDir, name));
|
||||
if (opts.gc) {
|
||||
const orphans = await orphanMembers(slots);
|
||||
for (const slot of [...orphans]) {
|
||||
// An analyze holds its slot's index lock until it has registered the
|
||||
// checkout, so a slot that is seeded but not yet registered is busy,
|
||||
// not orphaned. Judge (and, unless previewing, delete) only a slot
|
||||
// whose lock is free, so the preview matches what --force would do.
|
||||
let lock: IndexLockHandle;
|
||||
let gate: IndexLockHandle | undefined;
|
||||
try {
|
||||
gate = await acquireIndexLock(sharedMemberGatePath(storeRoot, slot), {
|
||||
timeoutMs: 1,
|
||||
sweep: false,
|
||||
});
|
||||
requireExclusiveIndexLock(gate, `Cannot acquire the shared member gate at ${slot}.`);
|
||||
// Recheck durable work under ownership before deleting anything. In
|
||||
// particular, acquiring the lock must not sweep a just-prepared graph.
|
||||
lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: false });
|
||||
} catch {
|
||||
gate?.release();
|
||||
orphans.delete(slot);
|
||||
continue;
|
||||
}
|
||||
if (!opts.dryRun) {
|
||||
try {
|
||||
await fs.rm(slot, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Like an undeletable graph below: keep it for the next collection
|
||||
// rather than abort every store after this one. It stays a member,
|
||||
// so whatever graph it still names is kept too.
|
||||
const branchLocks: IndexLockHandle[] = [];
|
||||
let dropped = false;
|
||||
let deleting = false;
|
||||
try {
|
||||
let busy = lock.lockFree;
|
||||
if (!busy) {
|
||||
for (const branch of (await memberMetadataDirectories(slot)).slice(1)) {
|
||||
try {
|
||||
const branchLock = await acquireIndexLock(branch, { timeoutMs: 1, sweep: false });
|
||||
branchLocks.push(branchLock);
|
||||
if (branchLock.lockFree) {
|
||||
busy = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
busy = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (busy || !(await orphanMembers([slot])).has(slot)) {
|
||||
orphans.delete(slot);
|
||||
result.keptMembers.push(slot);
|
||||
continue;
|
||||
}
|
||||
if (!opts.dryRun) {
|
||||
try {
|
||||
deleting = true;
|
||||
await fs.rm(slot, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Like an undeletable graph below: keep it for the next collection
|
||||
// rather than abort every store after this one. It stays a member,
|
||||
// so whatever graph it still names is kept too.
|
||||
orphans.delete(slot);
|
||||
result.keptMembers.push(slot);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
result.droppedMembers.push(slot);
|
||||
dropped = true;
|
||||
} finally {
|
||||
for (const branchLock of branchLocks) branchLock.release();
|
||||
lock.release();
|
||||
// A deleted member must not be recreated until all graph and store
|
||||
// deletion finishes. This gate is outside the recursively removed tree.
|
||||
if (dropped || deleting) heldMemberGates.push(gate);
|
||||
else gate.release();
|
||||
}
|
||||
result.droppedMembers.push(slot);
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
slots = slots.filter((slot) => !orphans.has(slot));
|
||||
}
|
||||
for (const slot of slots) {
|
||||
for (const directory of await memberMetadataDirectories(slot)) {
|
||||
const graphPath = (await loadMetaStrict(directory))?.graphPath;
|
||||
if (graphPath) referenced.add(path.dirname(path.resolve(directory, graphPath)));
|
||||
}
|
||||
}
|
||||
slots = slots.filter((slot) => !orphans.has(slot));
|
||||
}
|
||||
for (const slot of slots) {
|
||||
const graphPath = (await loadMetaStrict(slot))?.graphPath;
|
||||
if (graphPath) referenced.add(path.dirname(path.resolve(graphPath)));
|
||||
}
|
||||
|
||||
for (const name of await listDirStrict(commitsDir)) {
|
||||
const dir = path.join(commitsDir, name);
|
||||
if (referenced.has(dir)) continue;
|
||||
if (opts.dryRun) {
|
||||
if (!name.startsWith('.')) result.removed.push(dir);
|
||||
continue;
|
||||
for (const name of await listDirStrict(commitsDir)) {
|
||||
const dir = path.join(commitsDir, name);
|
||||
if (referenced.has(dir)) continue;
|
||||
if (opts.dryRun) {
|
||||
if (!name.startsWith('.')) result.removed.push(dir);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
if (!name.startsWith('.')) result.removed.push(dir);
|
||||
} catch {
|
||||
// Windows refuses to delete a file another process has open (an MCP
|
||||
// reader). Keep it for the next reclaim instead of failing the caller.
|
||||
if (!name.startsWith('.')) result.kept.push(dir);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
if (!name.startsWith('.')) result.removed.push(dir);
|
||||
} catch {
|
||||
// Windows refuses to delete a file another process has open (an MCP
|
||||
// reader). Keep it for the next reclaim instead of failing the caller.
|
||||
if (!name.startsWith('.')) result.kept.push(dir);
|
||||
}
|
||||
}
|
||||
|
||||
const isEmpty = async (): Promise<boolean> =>
|
||||
(await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0;
|
||||
if (!opts.dryRun && (await isEmpty())) {
|
||||
// Also hold the cache lock (publish -> cache, the only nesting order) so
|
||||
// a member saving caches cannot lose them, then re-check: a new member's
|
||||
// slot may have appeared while waiting. The lock directories live inside
|
||||
// the store; removing them while held is safe on POSIX and is retried on
|
||||
// the next reclaim elsewhere.
|
||||
await withStoreLock({ root: storeRoot }, 'cache', async () => {
|
||||
if (!(await isEmpty())) return;
|
||||
await fs
|
||||
.rm(storeRoot, { recursive: true, force: true })
|
||||
.then(() => {
|
||||
result.storeRemoved = true;
|
||||
})
|
||||
.catch(() => {});
|
||||
});
|
||||
const isEmpty = async (): Promise<boolean> =>
|
||||
(await listDirStrict(checkoutsDir)).length + (await listDirStrict(commitsDir)).length === 0;
|
||||
if (!opts.dryRun && (await isEmpty())) {
|
||||
// Also hold the cache lock (publish -> cache, the only nesting order) so
|
||||
// a member saving caches cannot lose them, then re-check: a new member's
|
||||
// slot may have appeared while waiting. The lock directories live inside
|
||||
// the store; removing them while held is safe on POSIX and is retried on
|
||||
// the next reclaim elsewhere.
|
||||
await withStoreLock({ root: storeRoot }, 'cache', async () => {
|
||||
if (!(await isEmpty())) return;
|
||||
await fs
|
||||
.rm(storeRoot, { recursive: true, force: true })
|
||||
.then(() => {
|
||||
result.storeRemoved = true;
|
||||
})
|
||||
.catch(() => {});
|
||||
});
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
for (const gate of heldMemberGates) gate.release();
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -292,9 +373,7 @@ export const describeSharedGraph = (
|
|||
graphPath: string,
|
||||
storagePath: string,
|
||||
): 'shared' | 'private' =>
|
||||
path.resolve(graphPath) === path.join(path.resolve(storagePath), LBUG_DIRECTORY)
|
||||
? 'private'
|
||||
: 'shared';
|
||||
path.dirname(path.resolve(graphPath)) === path.resolve(storagePath) ? 'private' : 'shared';
|
||||
|
||||
/** Files a shared checkout keeps in `<checkout>/.gitnexus`; everything else there is legacy. */
|
||||
const POINTER_DIR_KEEP = new Set([SHARED_STORE_POINTER, '.gitignore', 'run.cjs']);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import { stripWindowsLongPathPrefix } from '../lib/utils.js';
|
|||
import { getGlobalDir } from './global-dir.js';
|
||||
import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from './storage-constants.js';
|
||||
import { slotNameForCanonicalPath, STORAGE_PATH_ENV, STORAGE_ROOT_ENV } from './storage-slot.js';
|
||||
import type { RepoMeta } from './repo-meta.js';
|
||||
|
||||
export const SHARED_STORE_ENV = 'GITNEXUS_SHARED_STORE';
|
||||
export const STORES_DIR = 'stores';
|
||||
|
|
@ -224,10 +225,10 @@ export const storeRootOfCheckoutSlot = (storagePath: string): string | null => {
|
|||
};
|
||||
|
||||
/**
|
||||
* The graph a flat storage slot reads.
|
||||
* The graph a storage slot reads.
|
||||
*
|
||||
* Non-shared storage is always `<storagePath>/lbug`, with no I/O. A shared
|
||||
* checkout slot may record `graphPath` in its metadata, naming a commit graph
|
||||
* A private generation is an owned basename in the slot. A shared checkout
|
||||
* slot may record `graphPath` in its metadata, naming a commit graph
|
||||
* in the same store; any other recorded value (outside the store, a sibling's
|
||||
* private slot, unreadable metadata) falls back to the slot's own graph so a
|
||||
* hand-edited file cannot redirect reads.
|
||||
|
|
@ -244,13 +245,31 @@ const readRecordedGraphPath = (slot: string): unknown => {
|
|||
const metaPath = path.resolve(root, INDEX_METADATA_FILE);
|
||||
const rel = path.relative(root, metaPath);
|
||||
if (rel.startsWith('..') || path.isAbsolute(rel)) return undefined;
|
||||
const cached = recordedGraphCache.get(metaPath);
|
||||
if (cached) {
|
||||
const current = fs.lstatSync(metaPath);
|
||||
if (!current.isFile()) throw new Error('Index metadata must be a regular file');
|
||||
if (
|
||||
cached.key ===
|
||||
`${current.dev}:${current.ino}:${current.size}:${current.mtimeMs}:${current.ctimeMs}`
|
||||
)
|
||||
return cached.recorded;
|
||||
}
|
||||
// One open, then stat and read through the same descriptor, so the cache
|
||||
// key always describes the bytes that were parsed.
|
||||
const fd = fs.openSync(metaPath, 'r');
|
||||
const fd = fs.openSync(
|
||||
metaPath,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0),
|
||||
);
|
||||
try {
|
||||
const stat = fs.fstatSync(fd);
|
||||
const key = `${stat.ino}:${stat.size}:${stat.mtimeMs}`;
|
||||
const cached = recordedGraphCache.get(metaPath);
|
||||
if (!stat.isFile()) throw new Error('Index metadata must be a regular file');
|
||||
if (!fs.constants.O_NOFOLLOW) {
|
||||
const entry = fs.lstatSync(metaPath);
|
||||
if (!entry.isFile() || entry.ino !== stat.ino || entry.dev !== stat.dev)
|
||||
throw new Error('Index metadata changed during open');
|
||||
}
|
||||
const key = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
|
||||
if (cached?.key === key) return cached.recorded;
|
||||
const recorded = (JSON.parse(fs.readFileSync(fd, 'utf-8')) as { graphPath?: unknown })
|
||||
.graphPath;
|
||||
|
|
@ -261,17 +280,37 @@ const readRecordedGraphPath = (slot: string): unknown => {
|
|||
}
|
||||
};
|
||||
|
||||
export const resolveGraphPath = (storagePath: string): string => {
|
||||
export const resolveGraphPath = (
|
||||
storagePath: string,
|
||||
metadata?: Pick<RepoMeta, 'graphPath'>,
|
||||
): string => {
|
||||
const own = path.join(storagePath, LBUG_DIRECTORY);
|
||||
const root = storeRootOfCheckoutSlot(storagePath);
|
||||
if (!root) return own;
|
||||
let recorded: unknown;
|
||||
try {
|
||||
recorded = readRecordedGraphPath(storagePath);
|
||||
// Capability readers already loaded this receipt; validate that same
|
||||
// snapshot without another stat, read, or parse on the status hot path.
|
||||
recorded = metadata === undefined ? readRecordedGraphPath(storagePath) : metadata.graphPath;
|
||||
} catch {
|
||||
return own;
|
||||
}
|
||||
if (typeof recorded !== 'string' || !path.isAbsolute(recorded)) return own;
|
||||
if (typeof recorded !== 'string') return own;
|
||||
if (/^lbug\.generation\.[a-f0-9]{32}$/.test(recorded)) {
|
||||
const graph = path.join(storagePath, recorded);
|
||||
const relative = path.relative(storagePath, graph);
|
||||
if (relative.startsWith('..') || path.isAbsolute(relative)) return own;
|
||||
// Never follow an edited pointer to a symlink or another slot. A missing
|
||||
// committed generation must fail at open, rather than serve an older lbug.
|
||||
try {
|
||||
const stat = fs.lstatSync(graph);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Published graph generation must be an owned regular file');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
const root = storeRootOfCheckoutSlot(storagePath);
|
||||
if (!root || !path.isAbsolute(recorded)) return own;
|
||||
const graph = path.resolve(recorded);
|
||||
const commitDir = path.dirname(graph);
|
||||
// Only a published `<commit>-<featureKey>` dir, never `.publish-*` staging.
|
||||
|
|
@ -282,6 +321,15 @@ export const resolveGraphPath = (storagePath: string): string => {
|
|||
return valid ? graph : own;
|
||||
};
|
||||
|
||||
/** A private generation remains its own write target. Only an immutable
|
||||
* shared commit graph needs a new slot-local file before mutation. */
|
||||
export const resolveGraphWritePath = (storagePath: string): string => {
|
||||
const graph = resolveGraphPath(storagePath);
|
||||
return path.dirname(path.resolve(graph)) === path.resolve(storagePath)
|
||||
? graph
|
||||
: path.join(storagePath, LBUG_DIRECTORY);
|
||||
};
|
||||
|
||||
/**
|
||||
* The store slot named by `<checkout>/.gitnexus/store.json`, or null. The
|
||||
* recorded slot must be this checkout's own slot under the stores directory,
|
||||
|
|
|
|||
191
gitnexus/src/storage/shared-vector-artifact.ts
Normal file
191
gitnexus/src/storage/shared-vector-artifact.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
/** Retain a completed, immutable vector file in another owned slot.
|
||||
* Caller holds the source/destination locks (or the shared publish lock).
|
||||
* Pending artifacts, manifests, journals and mutable ownership never move. */
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants, type BigIntStats } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
embeddingJobDirectory,
|
||||
readEmbeddingJobRecord,
|
||||
syncEmbeddingJobDirectory,
|
||||
writeEmbeddingJobRecord,
|
||||
} from './embedding-job.js';
|
||||
import {
|
||||
assertOwnedVectorGeneration,
|
||||
embeddingGenerationDirectory,
|
||||
type PublishedVectorGeneration,
|
||||
} from './vector-generation.js';
|
||||
|
||||
type FileIdentity = { dev: string; ino: string; size: string; mtimeNs: string };
|
||||
interface RetentionReceipt {
|
||||
version: 1;
|
||||
identity?: string;
|
||||
signature: string;
|
||||
source: FileIdentity;
|
||||
/** Absent until the entire retained file is durable. */
|
||||
file?: FileIdentity;
|
||||
}
|
||||
const facts = (stat: BigIntStats): FileIdentity => ({
|
||||
dev: String(stat.dev),
|
||||
ino: String(stat.ino),
|
||||
size: String(stat.size),
|
||||
mtimeNs: String(stat.mtimeNs),
|
||||
});
|
||||
const sameFile = (a: FileIdentity | undefined, b: FileIdentity): boolean =>
|
||||
!!a && (['dev', 'ino', 'size', 'mtimeNs'] as const).every((key) => a[key] === b[key]);
|
||||
|
||||
async function completedFile(file: string): Promise<FileIdentity> {
|
||||
const stat = await fs.lstat(file, { bigint: true });
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Retained vectors must be an owned regular file');
|
||||
for (const suffix of ['.wal', '.shadow']) {
|
||||
try {
|
||||
const sidecar = await fs.lstat(file + suffix);
|
||||
if (!sidecar.isFile() || sidecar.isSymbolicLink() || sidecar.size)
|
||||
throw new Error('Cannot retain vectors with unfinished native writes');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
return facts(stat);
|
||||
}
|
||||
|
||||
async function receipt(directory: string, signature: string): Promise<RetentionReceipt> {
|
||||
const value = (await readEmbeddingJobRecord(directory, 'retained.json')) as RetentionReceipt;
|
||||
if (!value || value.version !== 1 || value.signature !== signature || !value.source)
|
||||
throw new Error('Unverified vector retention receipt; artifacts retained');
|
||||
return value;
|
||||
}
|
||||
|
||||
/** A crash can leave the directory before its first atomic receipt rename.
|
||||
* Only unpublished receipt temporaries are ours to discard in that state. */
|
||||
async function recoverEmptyPreparation(directory: string): Promise<void> {
|
||||
await embeddingJobDirectory(directory);
|
||||
const temporary = /^\.retained\.json\.[a-f0-9-]{36}\.tmp$/;
|
||||
for await (const entry of await fs.opendir(directory)) {
|
||||
const stat = await fs.lstat(path.join(directory, entry.name));
|
||||
if (!temporary.test(entry.name) || !stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Unverified vector preparation directory; artifacts retained');
|
||||
}
|
||||
// Validate the whole directory before removing anything; stream both passes
|
||||
// so repeated interrupted attempts cannot require an unbounded entry array.
|
||||
for await (const entry of await fs.opendir(directory)) {
|
||||
await fs.unlink(path.join(directory, entry.name));
|
||||
}
|
||||
}
|
||||
|
||||
/** Hard links pin the inode independently of source cleanup. Cross-device
|
||||
* copies are allowed only for initial local-index adoption, never rebases.
|
||||
* A directly named preparation directory makes an interrupted copy resumable
|
||||
* without scanning old jobs or overwriting an unverified existing artifact. */
|
||||
export async function retainSharedVectorArtifact(
|
||||
sourceRoot: string,
|
||||
destinationRoot: string,
|
||||
generation: PublishedVectorGeneration,
|
||||
allowCopy = false,
|
||||
): Promise<void> {
|
||||
const artifact = generation.artifactIdentity ?? generation.identity;
|
||||
const source = await assertOwnedVectorGeneration(sourceRoot, artifact);
|
||||
const sourceIdentity = await completedFile(source);
|
||||
const root = await embeddingJobDirectory(destinationRoot);
|
||||
const directory = embeddingGenerationDirectory(root, artifact);
|
||||
const parent = path.dirname(directory);
|
||||
try {
|
||||
await fs.mkdir(parent, { mode: 0o700 });
|
||||
await syncEmbeddingJobDirectory(root);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
||||
}
|
||||
await embeddingJobDirectory(parent);
|
||||
const signature = createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify([
|
||||
artifact,
|
||||
generation.vectorIdentity,
|
||||
generation.inputIdentity,
|
||||
generation.dimensions,
|
||||
generation.nodes,
|
||||
generation.chunks,
|
||||
generation.indexKind,
|
||||
]),
|
||||
)
|
||||
.digest('hex');
|
||||
|
||||
let present = false;
|
||||
try {
|
||||
await fs.lstat(directory);
|
||||
present = true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
if (present) {
|
||||
const file = await assertOwnedVectorGeneration(root, artifact);
|
||||
const identity = await completedFile(file);
|
||||
if (
|
||||
file === source ||
|
||||
(sourceIdentity.dev !== '0' &&
|
||||
sourceIdentity.ino !== '0' &&
|
||||
sameFile(identity, sourceIdentity))
|
||||
)
|
||||
return;
|
||||
const retained = await receipt(directory, signature);
|
||||
if (!sameFile(retained.file, identity))
|
||||
throw new Error('Retained vector file changed; artifacts retained');
|
||||
return;
|
||||
}
|
||||
|
||||
const preparing = path.join(parent, `.retain-${artifact}`);
|
||||
let intent: RetentionReceipt;
|
||||
try {
|
||||
await fs.mkdir(preparing, { mode: 0o700 });
|
||||
intent = { version: 1, identity: artifact, signature, source: sourceIdentity };
|
||||
await writeEmbeddingJobRecord(preparing, 'retained.json', intent);
|
||||
await syncEmbeddingJobDirectory(parent);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
||||
try {
|
||||
intent = await receipt(preparing, signature);
|
||||
} catch (readError) {
|
||||
if ((readError as NodeJS.ErrnoException).code !== 'ENOENT') throw readError;
|
||||
await recoverEmptyPreparation(preparing);
|
||||
intent = { version: 1, identity: artifact, signature, source: sourceIdentity };
|
||||
await writeEmbeddingJobRecord(preparing, 'retained.json', intent);
|
||||
await syncEmbeddingJobDirectory(parent);
|
||||
}
|
||||
if (!sameFile(intent.source, sourceIdentity))
|
||||
throw new Error('Vector retention source changed; artifacts retained');
|
||||
}
|
||||
const target = path.join(preparing, 'vectors');
|
||||
if (intent.file) {
|
||||
if (!sameFile(intent.file, await completedFile(target)))
|
||||
throw new Error('Prepared vector file changed; artifacts retained');
|
||||
} else {
|
||||
// A verified intent owns only this unacknowledged preparation file.
|
||||
// Its immutable source remains available even if a copy was interrupted.
|
||||
try {
|
||||
await completedFile(target);
|
||||
await fs.unlink(target);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
try {
|
||||
await fs.link(source, target);
|
||||
} catch (error) {
|
||||
if (!allowCopy || (error as NodeJS.ErrnoException).code !== 'EXDEV') throw error;
|
||||
await fs.copyFile(source, target, constants.COPYFILE_EXCL);
|
||||
const handle = await fs.open(target, 'r+');
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
if (!sameFile(sourceIdentity, await completedFile(source)))
|
||||
throw new Error('Vector source changed during retention; artifacts retained');
|
||||
intent = { ...intent, file: await completedFile(target) };
|
||||
await writeEmbeddingJobRecord(preparing, 'retained.json', intent);
|
||||
}
|
||||
await fs.rename(preparing, directory);
|
||||
await syncEmbeddingJobDirectory(parent);
|
||||
}
|
||||
|
|
@ -9,11 +9,11 @@
|
|||
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { BRANCHES_DIR } from './branch-index.js';
|
||||
import { BRANCHES_DIR, branchSlug } from './branch-index.js';
|
||||
import { listLocalHeads } from './git.js';
|
||||
import { isMissingFilesystemError, loadMeta } from './repo-meta.js';
|
||||
import { mapPool } from './map-pool.js';
|
||||
import { getStoragePaths, removeBranchIndex } from './repo-manager.js';
|
||||
import { removeBranchIndex } from './repo-manager.js';
|
||||
|
||||
export type StaleBranchReason =
|
||||
| 'ref-missing'
|
||||
|
|
@ -40,8 +40,8 @@ export interface ListStaleBranchSlotsInput {
|
|||
includeSize?: boolean;
|
||||
}
|
||||
|
||||
const slotDirForBranch = (repoPath: string, storagePath: string, branch: string): string =>
|
||||
path.dirname(getStoragePaths(repoPath, branch, storagePath).metaPath);
|
||||
const slotDirForBranch = (storagePath: string, branch: string): string =>
|
||||
path.join(storagePath, BRANCHES_DIR, branchSlug(branch));
|
||||
|
||||
/** Same bound as `listRegisteredRepos`: cap concurrent slot I/O. */
|
||||
const STALE_SLOT_IO_CONCURRENCY = 8;
|
||||
|
|
@ -191,10 +191,7 @@ export const listStaleBranchSlots = async (
|
|||
|
||||
const registryByDir = new Map<string, string>();
|
||||
for (const row of recorded) {
|
||||
registryByDir.set(
|
||||
path.resolve(slotDirForBranch(input.repoPath, input.storagePath, row.branch)),
|
||||
row.branch,
|
||||
);
|
||||
registryByDir.set(path.resolve(slotDirForBranch(input.storagePath, row.branch)), row.branch);
|
||||
}
|
||||
|
||||
const branchesProbe = await probeContainedBranchesRoot(input.storagePath);
|
||||
|
|
@ -519,7 +516,7 @@ export const removeBranchSlot = async (
|
|||
}
|
||||
|
||||
const dropRegistry =
|
||||
dir === null || isSameNormalizedPath(dir, slotDirForBranch(repoPath, storagePath, branch));
|
||||
dir === null || isSameNormalizedPath(dir, slotDirForBranch(storagePath, branch));
|
||||
if (dropRegistry) {
|
||||
try {
|
||||
await removeBranchIndex(repoPath, branch);
|
||||
|
|
|
|||
146
gitnexus/src/storage/vector-generation.ts
Normal file
146
gitnexus/src/storage/vector-generation.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
/** Filesystem-only vector routing. No DB or embedding runtime is loaded here. */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { RepoMeta } from './repo-meta.js';
|
||||
|
||||
export type PublishedVectorGeneration = NonNullable<RepoMeta['vectorGeneration']>;
|
||||
export type VectorReadSelection =
|
||||
| { kind: 'legacy' }
|
||||
| { kind: 'unavailable'; reason: string }
|
||||
| { kind: 'ready'; path: string; generation: PublishedVectorGeneration; coverage?: 'preserved' };
|
||||
const DIGEST = /^[a-f0-9]{64}$/;
|
||||
const GENERATION = /^[a-f0-9]{32}$/;
|
||||
const nonnegative = (n: unknown): n is number =>
|
||||
typeof n === 'number' && Number.isSafeInteger(n) && n >= 0;
|
||||
|
||||
export function embeddingGenerationDirectory(metadataDirectory: string, identity: string): string {
|
||||
if (!DIGEST.test(identity)) throw new Error('Invalid embedding generation identity');
|
||||
return path.join(metadataDirectory, 'embeddings', identity);
|
||||
}
|
||||
|
||||
/** Metadata must describe the exact graph handle used for result hydration. */
|
||||
export function metadataMatchesGraph(
|
||||
metadataDirectory: string,
|
||||
metadata: RepoMeta,
|
||||
graphPath: string,
|
||||
): boolean {
|
||||
const recorded = metadata.graphPath;
|
||||
if (recorded !== undefined && typeof recorded !== 'string') return false;
|
||||
return path.resolve(metadataDirectory, recorded ?? 'lbug') === path.resolve(graphPath);
|
||||
}
|
||||
|
||||
/** Advance only a previously bound completed reference. Called under the slot
|
||||
* lock before invalidating an in-place graph, or while preparing a staged one.
|
||||
* It neither opens vectors nor upgrades missing/foreign coverage provenance. */
|
||||
export function preserveVectorCoverage(
|
||||
metadata: RepoMeta,
|
||||
nextGeneration: string,
|
||||
): RepoMeta['vectorGeneration'] {
|
||||
if (!GENERATION.test(nextGeneration)) throw new Error('Invalid preservation target');
|
||||
const vector = metadata.vectorGeneration;
|
||||
const graph = metadata.graphGeneration;
|
||||
if (
|
||||
!vector ||
|
||||
vector.version !== 1 ||
|
||||
!GENERATION.test(vector.graphGeneration) ||
|
||||
!graph ||
|
||||
graph.version !== 1 ||
|
||||
!GENERATION.test(graph.id) ||
|
||||
(vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id)
|
||||
)
|
||||
return vector;
|
||||
return { ...vector, preservedForGraph: nextGeneration };
|
||||
}
|
||||
|
||||
/** Run before model initialization, vector COUNTs, or opening an artifact. */
|
||||
export function selectVectorGeneration(
|
||||
metadataDirectory: string,
|
||||
metadata: RepoMeta | null | undefined,
|
||||
graphPath: string,
|
||||
exactScanLimit: number,
|
||||
): VectorReadSelection {
|
||||
if (metadata?.incrementalInProgress || metadata?.graphWriteCollapsed)
|
||||
return { kind: 'unavailable', reason: 'graph-generation-unverified' };
|
||||
if (!metadata?.graphGeneration && !metadata?.vectorGeneration && !metadata?.embeddingJob)
|
||||
return { kind: 'legacy' };
|
||||
const graph = metadata?.graphGeneration;
|
||||
const vector = metadata?.vectorGeneration;
|
||||
if (
|
||||
!graph ||
|
||||
graph.version !== 1 ||
|
||||
!GENERATION.test(graph.id) ||
|
||||
!DIGEST.test(graph.coveredContent) ||
|
||||
!metadataMatchesGraph(metadataDirectory, metadata, graphPath)
|
||||
)
|
||||
return { kind: 'unavailable', reason: 'graph-generation-unverified' };
|
||||
if (
|
||||
!vector &&
|
||||
!metadata.embeddingJob &&
|
||||
typeof metadata.legacyVectorSource === 'string' &&
|
||||
DIGEST.test(metadata.legacyVectorSource) &&
|
||||
nonnegative(metadata.stats?.embeddings) &&
|
||||
metadata.stats!.embeddings! > 0 &&
|
||||
nonnegative(metadata.embeddingDims) &&
|
||||
metadata.embeddingDims > 0
|
||||
)
|
||||
return { kind: 'legacy' };
|
||||
if (!vector)
|
||||
return {
|
||||
kind: 'unavailable',
|
||||
reason: metadata.embeddingJob ? 'embedding-pending' : 'embeddings-unavailable',
|
||||
};
|
||||
if (metadata.embeddingJob) return { kind: 'unavailable', reason: 'embedding-pending' };
|
||||
if (
|
||||
vector.version !== 1 ||
|
||||
!GENERATION.test(vector.graphGeneration) ||
|
||||
(vector.graphGeneration !== graph.id && vector.preservedForGraph !== graph.id) ||
|
||||
(vector.preservedForGraph !== undefined && !GENERATION.test(vector.preservedForGraph)) ||
|
||||
!DIGEST.test(vector.identity) ||
|
||||
(vector.artifactIdentity !== undefined && !DIGEST.test(vector.artifactIdentity)) ||
|
||||
!DIGEST.test(vector.vectorIdentity) ||
|
||||
!DIGEST.test(vector.inputIdentity) ||
|
||||
!nonnegative(vector.dimensions) ||
|
||||
!vector.dimensions ||
|
||||
!nonnegative(vector.nodes) ||
|
||||
(vector.excludedNodes !== undefined &&
|
||||
(!nonnegative(vector.excludedNodes) ||
|
||||
vector.excludedNodes > vector.nodes ||
|
||||
vector.exclusionReason !== 'input-rejected')) ||
|
||||
!nonnegative(vector.chunks) ||
|
||||
vector.chunks < vector.nodes - (vector.excludedNodes ?? 0) ||
|
||||
(vector.indexKind !== 'hnsw' && vector.indexKind !== 'exact')
|
||||
)
|
||||
return { kind: 'unavailable', reason: 'vector-generation-mismatch' };
|
||||
if (!vector.chunks) return { kind: 'unavailable', reason: 'embeddings-empty' };
|
||||
if (vector.indexKind === 'exact' && vector.chunks > exactScanLimit)
|
||||
return { kind: 'unavailable', reason: 'vector-index-pending' };
|
||||
return {
|
||||
kind: 'ready',
|
||||
path: path.join(
|
||||
embeddingGenerationDirectory(metadataDirectory, vector.artifactIdentity ?? vector.identity),
|
||||
'vectors',
|
||||
),
|
||||
generation: vector,
|
||||
...(vector.graphGeneration === graph.id ? {} : { coverage: 'preserved' as const }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Refuse symlinks from the owned metadata slot down to the database. Parent
|
||||
* path aliases remain valid. The coordinator holds the slot lock while creating
|
||||
* jobs; published directories are immutable. */
|
||||
export async function assertOwnedVectorGeneration(
|
||||
metadataDirectory: string,
|
||||
identity: string,
|
||||
): Promise<string> {
|
||||
const directory = embeddingGenerationDirectory(metadataDirectory, identity);
|
||||
for (const target of [metadataDirectory, path.dirname(directory), directory]) {
|
||||
const stat = await fs.lstat(target);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink())
|
||||
throw new Error('Vector generation must be an owned directory');
|
||||
}
|
||||
const file = path.join(directory, 'vectors');
|
||||
const stat = await fs.lstat(file);
|
||||
if (!stat.isFile() || stat.isSymbolicLink())
|
||||
throw new Error('Vector generation must be an owned regular file');
|
||||
return file;
|
||||
}
|
||||
11
gitnexus/test/helpers/analysis-http-server.mjs
Normal file
11
gitnexus/test/helpers/analysis-http-server.mjs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import http from 'node:http';
|
||||
const listen = http.Server.prototype.listen;
|
||||
http.Server.prototype.listen = function (...args) {
|
||||
this.once('listening', () => process.send?.({ port: this.address().port }));
|
||||
return listen.apply(this, args);
|
||||
};
|
||||
const { createServer } = await import('../../dist/server/api.js');
|
||||
await createServer(0);
|
||||
process.on('message', (message) => {
|
||||
if (message === 'shutdown') process.emit('SIGTERM');
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue