From 07b5c27045bffd052b864e1c94bdfe806069da7c Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Sat, 3 Oct 2026 12:09:27 +0100 Subject: [PATCH] fix(embeddings): defer staged checkpoint count until publication --- gitnexus/src/core/run-analyze.ts | 24 +++++++++++-------- .../test/unit/run-analyze-fts-repair.test.ts | 19 ++++++++------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 92925de71..af51bb6ea 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -4472,6 +4472,7 @@ async function runFullAnalysisInner( semanticMode = vectorIndexReady ? 'vector-index' : 'exact-scan'; } + let stagedCheckpointEmbeddingCount: number | undefined; if (!embeddingSkipped) { const { isHttpMode } = await import('./embeddings/http-client.js'); const httpMode = isHttpMode(); @@ -4522,11 +4523,10 @@ async function runFullAnalysisInner( // /api/embed checkpoint writer in server/api.ts already uses, which also // keeps a concurrent writer's update from being reverted by a stale // snapshot) and replace ONLY `embeddingCheckpoint` — plus - // `stats.embeddings` when the caller actually MEASURED the live count - // (the post-window `onCheckpoint`). The window-start callback passes - // nothing: restating the previous run's count there both re-published a - // stale number and clobbered the live count a preceding `onCheckpoint` - // had just written. + // `stats.embeddings` when the caller actually MEASURED the published + // index (the post-window `onCheckpoint` on an in-place build). A staging + // build's count is not published until the atomic swap succeeds. The + // window-start callback passes nothing, preserving the latest count. const saveEmbeddingCheckpoint = async ( checkpoint: { nodesProcessed: number; @@ -4536,6 +4536,9 @@ async function runFullAnalysisInner( pendingNodeIds: string[], embeddings?: number, ): Promise => { + if (embeddings !== undefined && buildPath !== lbugPath) { + stagedCheckpointEmbeddingCount = embeddings; + } const latestMeta = (await loadMeta(metaDir)) ?? existingMeta; // First-ever analyze of this repo: no meta exists on disk yet (the // pre-wipe dirty stamp only fires when one does). Mint the minimum @@ -4549,7 +4552,9 @@ async function runFullAnalysisInner( }; await saveMeta(metaDir, { ...base, - ...(embeddings === undefined ? {} : { stats: { ...base.stats, embeddings } }), + ...(embeddings === undefined || buildPath !== lbugPath + ? {} + : { stats: { ...base.stats, embeddings } }), // Written by a run that is still IN FLIGHT — see the `kind` doc in // repo-manager.ts. embeddingCheckpoint: mintInterruptedCheckpoint( @@ -4752,14 +4757,13 @@ async function runFullAnalysisInner( // already written to disk: prior meta says 0, a clean run inserts // embeddings and checkpoints the real count, the final probe is // unavailable, and finalization carries the stale 0 forward while reporting - // success. `loadMeta` never throws (it returns null), and the checkpoint - // writer already re-reads the same way, so this is the same freshness - // discipline applied to the same field. + // success. For a staged build, use its last measured count only in the + // final meta, written after the swap; never publish it at a checkpoint. const latestMetaForCount = embeddingCount === undefined ? ((await loadMeta(metaDir)) ?? existingMeta) : undefined; const persistedEmbeddingCount = resolvePersistedEmbeddingCount( measuredEmbeddingCount, - latestMetaForCount?.stats?.embeddings, + stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, ); const { getRuntimeCapabilities } = await import('./platform/capabilities.js'); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 2f0304ef4..b6de84c6b 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -2215,10 +2215,11 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { vi.unstubAllEnvs(); }); - it('preserves lastCommit / fileHashes / the dirty flag, and never restates a stale count', async () => { + it('keeps staging counts out of published metadata until the index is swapped', async () => { const STALE_COMMIT = '1111111111111111111111111111111111111111'; const STALE_HASHES = { 'src/app.ts': 'stale-hash' }; const LIVE_EMBEDDING_COUNT = 42; + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '1'); vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ initLbug: vi.fn(async () => undefined), @@ -2315,15 +2316,14 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { nodeIds: ['node-1', 'node-2'], }); snapshots.windowStart = await loadMeta(storagePath); - // Post-window checkpoint — this one MEASURED the live count. + // Post-window checkpoint measures staging, not the published DB. await pipelineOptions.onCheckpoint?.({ nodesProcessed: 2, totalNodes: 4, chunksProcessed: 4, }); snapshots.postWindow = await loadMeta(storagePath); - // Window 2 — the old code restated the PREVIOUS run's count here and - // clobbered the live figure the post-window save had just written. + // Window 2 must retain the published count too. await pipelineOptions.onCheckpointWindowStart?.({ nodesProcessed: 2, totalNodes: 4, @@ -2375,18 +2375,18 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { }); expect(snapshots.windowStart?.lastCommit).not.toBe(currentCommit); - // ── Post-window: the one save that legitimately measured the count ── + // ── Post-window: the staged count is not published yet ───────────── expect(snapshots.postWindow).toMatchObject({ lastCommit: STALE_COMMIT, fileHashes: STALE_HASHES, incrementalInProgress: { phase: 'full-rebuild' }, - stats: { embeddings: LIVE_EMBEDDING_COUNT }, + stats: { embeddings: 7 }, }); - // ── Window 2: no stale restatement over the measured figure ──────── + // ── Window 2: the published count remains unchanged ──────────────── expect(snapshots.secondWindow).toMatchObject({ lastCommit: STALE_COMMIT, - stats: { embeddings: LIVE_EMBEDDING_COUNT }, + stats: { embeddings: 7 }, embeddingCheckpoint: { pendingNodeIds: ['node-3', 'node-4'] }, }); @@ -2395,7 +2395,7 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { const finalMeta = JSON.parse( await fs.readFile(`${storagePath}/meta.json`, 'utf-8'), ) as RepoMeta; - expect(finalMeta).toMatchObject({ lastCommit: currentCommit }); + expect(finalMeta).toMatchObject({ lastCommit: currentCommit, stats: { embeddings: 42 } }); expect(finalMeta.embeddingCheckpoint).toBeUndefined(); expect(finalMeta.incrementalInProgress).toBeUndefined(); } finally { @@ -2699,6 +2699,7 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => */ it('carries the mid-run count forward, not the run-start snapshot, so --force still loads the cache', async () => { const MID_RUN_COUNT = 12; + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '1'); const tmpRepo = await createTempDir('gitnexus-2790r-latest-meta-'); try { const { storagePath } = getStoragePaths(tmpRepo.dbPath);