mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-06 02:49:56 +00:00
fix(storage): keep graphs with pending embeddings out of the shared store (#3374)
Trigger: an analyze that finished with embeddings still owed (meta carries embeddingCheckpoint) was published as the commit graph, because the feature key ignores the checkpoint and publish never checked it. The published meta kept the checkpoint while the graph could never change. The next run healed the embeddings privately, found the commit graph already there, wiped its own healed graph and pointed back at the partial one. Fix: publishSharedGraph treats a checkpointed graph as not shareable, so it stays private and no published commit graph carries a checkpoint. When the target commit graph exists but records a checkpoint, has fewer stats.embeddings than the private graph, or has unreadable metadata, the checkout keeps its private graph instead of re-pointing. The commit graph is left as is because other checkouts may read it. Embeddings sync and the server embed job already privatize a pointer slot before writing (ensurePrivateSharedGraph). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
42c39b5c82
commit
82e01be5e9
2 changed files with 79 additions and 1 deletions
|
|
@ -370,6 +370,9 @@ export const publishSharedGraph = async (
|
|||
meta.lastCommit === currentCommit &&
|
||||
!meta.incrementalInProgress &&
|
||||
builtClean &&
|
||||
// Embeddings still owed stay with this checkout, which finishes them; a
|
||||
// commit graph never changes, so a shared copy would stay short for good.
|
||||
!meta.embeddingCheckpoint &&
|
||||
// A sparse or partial checkout builds a graph missing the files it hides.
|
||||
isWorkingTreePristine(repoPath);
|
||||
// Every pointer change and the reclaim that follows run under one publish
|
||||
|
|
@ -379,7 +382,23 @@ export const publishSharedGraph = async (
|
|||
const target = commitGraphDir(layout, currentCommit, featureKeyOf(meta));
|
||||
const targetGraph = path.join(target, LBUG_DIRECTORY);
|
||||
let published = await exists(targetGraph);
|
||||
if (published) {
|
||||
// A commit graph published before embedding checkpoints were kept
|
||||
// private may hold fewer embeddings than this checkout's own graph.
|
||||
// Keep the better private graph; the commit graph stays as it is, since
|
||||
// other checkouts may read it.
|
||||
const targetMeta = published ? await loadMeta(target) : null;
|
||||
const ownIsBetter =
|
||||
published &&
|
||||
(!targetMeta ||
|
||||
!!targetMeta.embeddingCheckpoint ||
|
||||
(targetMeta.stats?.embeddings ?? 0) < (meta.stats?.embeddings ?? 0)) &&
|
||||
(await exists(own));
|
||||
if (ownIsBetter) {
|
||||
published = false;
|
||||
log(
|
||||
`Shared store: commit graph ${currentCommit.slice(0, 12)} is less complete; keeping the private graph.`,
|
||||
);
|
||||
} else if (published) {
|
||||
try {
|
||||
await wipeLbugDbFiles(own);
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -390,4 +390,63 @@ describe('publishSharedGraph race (#3352)', () => {
|
|||
expect((await loadMeta(slot))?.graphPath).toBe(shared);
|
||||
expect(existsSync(path.join(slot, 'lbug'))).toBe(false);
|
||||
});
|
||||
|
||||
const checkpoint: NonNullable<RepoMeta['embeddingCheckpoint']> = {
|
||||
at: '2026-01-01T00:00:00.000Z',
|
||||
nodesProcessed: 1,
|
||||
totalNodes: 2,
|
||||
chunksProcessed: 1,
|
||||
model: 'm',
|
||||
dimensions: 4,
|
||||
provider: 'local',
|
||||
kind: 'partial',
|
||||
pendingNodeIds: ['n2'],
|
||||
};
|
||||
|
||||
// #3374: a graph with embeddings still owed would become every checkout's
|
||||
// graph, and its checkpoint-free copy would look complete forever.
|
||||
it('keeps a graph with pending embeddings private', async () => {
|
||||
const { checkouts, head } = await setup();
|
||||
const [main] = checkouts;
|
||||
const slot = layoutOf(main).checkoutSlot;
|
||||
const meta = (await loadMeta(slot)) as RepoMeta;
|
||||
await saveMeta(slot, { ...meta, embeddingCheckpoint: checkpoint });
|
||||
await publishSharedGraph(layoutOf(main), main, head, () => {});
|
||||
expect(await listCommitDirs(layoutOf(main))).toEqual([]);
|
||||
expect(existsSync(path.join(slot, 'lbug'))).toBe(true);
|
||||
expect((await loadMeta(slot))?.embeddingCheckpoint).toEqual(checkpoint);
|
||||
});
|
||||
|
||||
// A commit graph published before that rule may still be the weaker one;
|
||||
// the checkout keeps its own graph rather than trading down to it.
|
||||
it.each<[string, Partial<RepoMeta>]>([
|
||||
['records pending embeddings', { embeddingCheckpoint: checkpoint, stats: { embeddings: 5 } }],
|
||||
['has fewer embeddings', { stats: { embeddings: 2 } }],
|
||||
])('keeps the private graph when the published one %s', async (_label, targetDelta) => {
|
||||
const { checkouts, head } = await setup();
|
||||
const [main, wt] = checkouts;
|
||||
const layout = layoutOf(main);
|
||||
const slot = layout.checkoutSlot;
|
||||
const own: RepoMeta = { ...((await loadMeta(slot)) as RepoMeta), stats: { embeddings: 5 } };
|
||||
await saveMeta(slot, own);
|
||||
const target = commitGraphDir(layout, head, featureKeyOf(own));
|
||||
await fs.mkdir(target, { recursive: true });
|
||||
await fs.writeFile(path.join(target, 'lbug'), 'older published graph');
|
||||
const targetMeta: Partial<RepoMeta> = {
|
||||
lastCommit: head,
|
||||
indexedAt: own.indexedAt,
|
||||
...targetDelta,
|
||||
};
|
||||
await fs.writeFile(path.join(target, 'gitnexus.json'), JSON.stringify(targetMeta));
|
||||
// Another checkout reads the published graph.
|
||||
const wtSlot = layoutOf(wt).checkoutSlot;
|
||||
await fs.rm(path.join(wtSlot, 'lbug'));
|
||||
const wtMeta = (await loadMeta(wtSlot)) as RepoMeta;
|
||||
await saveMeta(wtSlot, { ...wtMeta, graphPath: path.join(target, 'lbug') });
|
||||
await publishSharedGraph(layout, main, head, () => {});
|
||||
expect(await fs.readFile(path.join(slot, 'lbug'), 'utf-8')).toBe(`graph from ${main}`);
|
||||
expect((await loadMeta(slot))?.graphPath).toBeUndefined();
|
||||
// The published graph is immutable: other checkouts may point at it.
|
||||
expect(await fs.readFile(path.join(target, 'lbug'), 'utf-8')).toBe('older published graph');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue