From e3fe9f9ab7da7be1ea0e4d13b61ba57ca62b8380 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 6 Nov 2026 09:04:08 +0000 Subject: [PATCH] fix(devcontainer): skip parser caches and report import progress Exclude regenerable GitNexus parser caches from every nested worktree while preserving graph databases, Git state, and local work. Report archive bytes and elapsed time every five seconds, and stream output with activity updates during longer Docker initialization stages. --- .devcontainer/README.md | 5 +- .devcontainer/initialize-volumes.cjs | 132 +++++++++++++++++----- .devcontainer/initialize-volumes.test.cjs | 87 +++++++++++++- .devcontainer/seed-workspace.cjs | 7 ++ .devcontainer/seed-workspace.test.cjs | 18 ++- 5 files changed, 218 insertions(+), 31 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 13635594a..327762491 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -24,7 +24,9 @@ Keep the host folder and selected configuration path stable: moving either can c The default workspace is `gitnexus-source-${devcontainerId}:/workspace`. The host `initializeCommand` runs `initialize-volumes.cjs`, starts a disposable helper with volume mounts only, and streams a tar archive through [Docker's copy API](https://docs.docker.com/reference/cli/docker/container/cp/). The helper uses the Dockerfile's digest-pinned base image. It is removed afterward; volumes and host files remain. -The import preserves `.git`, branches, local commits, the index, stashes, uncommitted and ignored files, checkout-local indexes, and nested worktrees. Generated `node_modules`, `.venv`, and `.husky/_` directories are excluded at every depth, including under `.worktrees` and `.claude/worktrees`; Git metadata is preserved except for source-volume completion markers. Reinstall dependencies inside each worktree as needed, and recreate Python environments with the evaluation project's `uv run` commands. Host files are left intact. Existing dependency, index-store, AI configuration, history, and session volume names stay unchanged. +The import preserves `.git`, branches, local commits, the index, stashes, uncommitted and ignored files, checkout-local indexes, and nested worktrees. Generated `node_modules`, `.venv`, `.husky/_`, `.gitnexus/parse-cache`, `.gitnexus/parse-cache.json`, and `.gitnexus/parsedfile-cache` paths are excluded at every depth, including under `.worktrees` and `.claude/worktrees`. GitNexus graph databases, embeddings, metadata, and runners are kept; parser caches regenerate on the next analysis. Git metadata is preserved except for source-volume completion markers. Reinstall dependencies inside each worktree as needed, and recreate Python environments with the evaluation project's `uv run` commands. Host files are left intact. Existing dependency, index-store, AI configuration, history, and session volume names stay unchanged. + +During archive transfers, `[volumes]` reports elapsed time, entries queued, excluded paths, symlinks, and MiB streamed to Docker every five seconds, followed by a completion or failure line. Streamed bytes measure data sent to Docker, not bytes committed to disk; there is no percentage because the total is not pre-scanned. Helper startup, the second copy into the source volume, Git normalization, snapshot finalization, and cleanup also report activity every five seconds. Output from the Linux helper is shown as it arrives. Node streams an explicit file list to tar with recursion disabled. Symlinks are transferred as path/target metadata and recreated by the Linux helper without following them. This also applies to configuration snapshots: Windows tar never has to inspect WSL-created links, which it can reject even when their paths are excluded. Restoration rejects paths outside the destination, symlinked parent directories, and replacements of existing files. Ordinary source links and their targets are preserved verbatim. @@ -290,6 +292,7 @@ If `[install-deps]` prints `npm warn EBADENGINE` with Node 22.16.0, run **Dev Co | --- | --- | | `node`, `docker`, or `tar` missing during initialization | Install the host prerequisite and restart VS Code to refresh PATH. | | Import helper already exists | Inspect `gitnexus-import-`; remove it only if the import is no longer running. | +| `no space left on device` during import | Docker's Linux storage is full. Check `docker system df` and Docker Desktop's disk allocation. Allow room for both the temporary `/seed` copy and the source volume, plus configuration snapshots. The importer now excludes regenerable parser caches at every depth. A failed `/seed` transfer is discarded with its helper, so retry after resolving space. If the failure occurred while copying into `/workspace`, inspect and preserve the partial source volume before resetting it. Do not prune volumes containing work. | | Uninitialized source volume / incomplete snapshot | Inspect and back up that volume, then retry with it empty. Never add a completion marker manually. | | 9P remains after rebuilding | Check the selected config, regenerate optional profiles, and inspect editor-added mounts and image metadata. | | Host edits no longer appear | Work in the attached container. Export files explicitly with Git or `docker cp`. | diff --git a/.devcontainer/initialize-volumes.cjs b/.devcontainer/initialize-volumes.cjs index e08e69877..ca335fe97 100644 --- a/.devcontainer/initialize-volumes.cjs +++ b/.devcontainer/initialize-volumes.cjs @@ -4,7 +4,7 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { execFileSync, spawn } = require('node:child_process'); -const { Readable } = require('node:stream'); +const { Readable, Transform } = require('node:stream'); const { pipeline } = require('node:stream/promises'); const { archiveEntries, @@ -83,47 +83,100 @@ function archiveArgs(source, entries, excludes = [], { recursive = true } = {}) ]; } +function startProgress(label, details = () => '', intervalMs = 5000) { + const started = performance.now(); + const report = (status) => { + const elapsed = ((performance.now() - started) / 1000).toFixed(1); + console.log(`[volumes] ${label}: ${status}; ${elapsed}s elapsed${details()}`); + }; + report('starting'); + const timer = setInterval(() => report('working'), intervalMs); + return (status) => { + clearInterval(timer); + report(status); + }; +} + +function childDone(child, name) { + return new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => + code === 0 ? resolve() : reject(new Error(`${name} exited ${signal || code}`)), + ); + }); +} + +async function runWithProgress(label, args, input) { + const finish = startProgress(label); + const child = spawn('docker', args, { + stdio: [input === undefined ? 'ignore' : 'pipe', 'inherit', 'inherit'], + }); + try { + const done = childDone(child, label); + const written = + input === undefined ? Promise.resolve() : pipeline(Readable.from([input]), child.stdin); + await Promise.all([done, written]); + finish('done'); + } catch (error) { + child.kill(); + finish('failed'); + throw error; + } +} + async function copyArchive(source, entries, destination, excludes = [], exclude = () => false) { if (!entries.length) return; // No shell: spaces, apostrophes and Windows drive letters remain literal arguments. // Watch both processes: a failed tar must never publish a successful seed marker. const links = []; + let queued = 0; + let bytes = 0; + let skipped = 0; + const finish = startProgress( + `Archive to ${destination.slice(destination.indexOf(':') + 1)}`, + () => + `; ${queued} entries queued, ${links.length} symlinks, ${skipped} excluded paths; ${(bytes / 1024 ** 2).toFixed(1)} MiB streamed to Docker`, + ); const listing = archiveEntries(source, entries, { links, - exclude: (relative) => - excludes.includes(relative.split(path.sep).join('/')) || exclude(relative), + exclude: (relative) => { + const omit = excludes.includes(relative.split(path.sep).join('/')) || exclude(relative); + if (omit) skipped++; + return omit; + }, }); const archive = spawn('tar', archiveArgs(source, [], [], { recursive: false }), { stdio: ['pipe', 'pipe', 'inherit'], }); const copy = spawn('docker', ['cp', '-', destination], { stdio: ['pipe', 'inherit', 'inherit'] }); - const done = (child, name) => - new Promise((resolve, reject) => { - child.once('error', reject); - child.once('close', (code) => - code === 0 ? resolve() : reject(new Error(`${name} exited ${code}`)), - ); - }); - const pipeDone = new Promise((resolve, reject) => { - copy.stdin.once('error', reject); - copy.stdin.once('finish', resolve); - archive.stdout.once('error', reject); + const meter = new Transform({ + transform(chunk, encoding, callback) { + bytes += chunk.length; + callback(null, chunk); + }, }); - archive.stdout.pipe(copy.stdin); try { const inputDone = pipeline( Readable.from( (function* () { - for (const entry of listing) yield `${entry}\0`; + for (const entry of listing) { + queued++; + yield `${entry}\0`; + } })(), ), archive.stdin, ); - await Promise.all([done(archive, 'tar'), done(copy, 'docker cp'), pipeDone, inputDone]); + await Promise.all([ + childDone(archive, 'tar'), + childDone(copy, 'docker cp'), + pipeline(archive.stdout, meter, copy.stdin), + inputDone, + ]); if (links.length) { const separator = destination.indexOf(':'); - execFileSync( - 'docker', + await runWithProgress( + 'Restoring symlinks', [ 'exec', '-i', @@ -133,12 +186,14 @@ async function copyArchive(source, entries, destination, excludes = [], exclude '--restore-links', destination.slice(separator + 1), ], - { input: JSON.stringify(links), stdio: ['pipe', 'inherit', 'inherit'] }, + JSON.stringify(links), ); } + finish('done'); } catch (error) { archive.kill(); copy.kill(); + finish('failed'); throw error; } } @@ -176,7 +231,7 @@ async function initialize( const name = `gitnexus-import-${id}`; let created = false; try { - docker( + await runWithProgress('Preparing import helper', [ 'create', '--name', name, @@ -189,20 +244,32 @@ async function initialize( 'sleep', image, 'infinity', - ); + ]); created = true; - docker('start', name); + await runWithProgress('Starting import helper', ['start', name]); for (const file of ['initialize-volumes.cjs', 'seed-workspace.cjs']) docker('cp', path.join(__dirname, file), `${name}:/tmp/${file}`); const run = (script, ...args) => docker('exec', name, 'node', `/tmp/${script}`, ...args); if (mode === 'source' && run('seed-workspace.cjs', '--state', '/workspace') === 'empty') { validateGitDirectory(path.join(workspace, '.git')); - console.log('[volumes] Importing the checkout, Git metadata and local work once...'); + console.log( + '[volumes] Importing the checkout, Git metadata and local work once; skipping dependencies and regenerable parser caches.', + ); + console.log( + '[volumes] Docker needs space for both the temporary import and the source volume.', + ); docker('exec', name, 'mkdir', '/seed'); // Windows tar stats WSL links before applying --exclude. Enumerate in Node // and disable tar recursion so it never encounters skipped dependencies. await copyArchive(workspace, ['.'], `${name}:/seed`, [], isWorkspaceExcluded); - run('seed-workspace.cjs', '/seed', '/workspace'); + await runWithProgress('Copying into the source volume and normalizing Git', [ + 'exec', + name, + 'node', + '/tmp/seed-workspace.cjs', + '/seed', + '/workspace', + ]); } // Linux hosts realign the remote UID; Desktop on Windows/macOS keeps node=1000. const uid = String(process.platform === 'linux' ? process.getuid() : 1000); @@ -224,12 +291,22 @@ async function initialize( SNAPSHOT_MARKER, ]); } - run('initialize-volumes.cjs', '--snapshot-finish', snapshot.target, uid, gid, state); + await runWithProgress(`Finalizing ${snapshot.target}`, [ + 'exec', + name, + 'node', + '/tmp/initialize-volumes.cjs', + '--snapshot-finish', + snapshot.target, + uid, + gid, + state, + ]); } console.log('[volumes] Linux volumes ready. Host files are not synchronized.'); } finally { // Only the disposable helper is removed, never any volume or host file. - if (created) docker('rm', '-f', name); + if (created) await runWithProgress('Removing temporary import helper', ['rm', '-f', name]); } } @@ -237,6 +314,7 @@ module.exports = { initialize, copyArchive, archiveArgs, + startProgress, snapshotState, finishSnapshot, HOST_PATHS, diff --git a/.devcontainer/initialize-volumes.test.cjs b/.devcontainer/initialize-volumes.test.cjs index 8a65fedab..5c4f67f95 100644 --- a/.devcontainer/initialize-volumes.test.cjs +++ b/.devcontainer/initialize-volumes.test.cjs @@ -14,12 +14,19 @@ function temporary(t) { return root; } -test('the host tar omits Node and Python dependencies while preserving ignored local files', (t) => { +test('the host tar omits dependencies and parser caches while preserving indexes and local files', (t) => { const root = temporary(t); for (const file of [ '.git/config', '.git/devcontainer-source-volume', '.git/refs/heads/node_modules/topic', + '.git/refs/heads/.gitnexus/parse-cache/topic', + '.gitnexus/parse-cache/chunk.v8', + '.gitnexus/parse-cache.json', + '.gitnexus/parsedfile-cache/chunk.v8', + '.gitnexus/lbug', + '.gitnexus/gitnexus.json', + '.gitnexus/run.cjs', '.env.local', '.husky/_/h', 'gitnexus/node_modules/binary', @@ -31,9 +38,15 @@ test('the host tar omits Node and Python dependencies while preserving ignored l '.worktrees/zig-receivers/gitnexus-shared/node_modules/binary', '.worktrees/zig-receivers/gitnexus-shared/src/index.ts', '.worktrees/zig-receivers/.husky/_/h', + '.worktrees/zig-receivers/.gitnexus/parse-cache/chunk.v8', + '.worktrees/zig-receivers/.gitnexus/parsedfile-cache/chunk.v8', + '.worktrees/zig-receivers/.gitnexus/lbug', '.claude/worktrees/topic/eval/.venv/bin/python', '.claude/worktrees/topic/local-edit', '.claude/worktrees/topic/.git', + '.claude/worktrees/topic/.gitnexus/parse-cache.json', + 'src/parse-cache/implementation.cjs', + '.gitnexus/parse-cache-notes/local-file', 'file with spaces', "file's name", '-leading-option', @@ -55,12 +68,44 @@ test('the host tar omits Node and Python dependencies while preserving ignored l assert.match(listing, /\.claude\/worktrees\/topic\/local-edit/); assert.match(listing, /\.claude\/worktrees\/topic\/\.git/); assert.match(listing, /\.git\/refs\/heads\/node_modules\/topic/); + assert.match(listing, /\.git\/refs\/heads\/\.gitnexus\/parse-cache\/topic/); + assert.match(listing, /\.gitnexus\/lbug/); + assert.match(listing, /\.gitnexus\/gitnexus.json/); + assert.match(listing, /\.gitnexus\/run.cjs/); + assert.match(listing, /\.worktrees\/zig-receivers\/\.gitnexus\/lbug/); + assert.match(listing, /src\/parse-cache\/implementation.cjs/); + assert.match(listing, /\.gitnexus\/parse-cache-notes\/local-file/); assert.doesNotMatch(listing, /eval\/\.venv(?:\/|$)/m); const workspace = listing .split(/\r?\n/) .filter((entry) => !entry.startsWith('./.git/')) .join('\n'); assert.doesNotMatch(workspace, /node_modules|devcontainer-source-volume|\.husky\/_/); + assert.doesNotMatch( + workspace, + /\.gitnexus\/(?:parse-cache(?:\.json)?|parsedfile-cache)(?:\/|$)/m, + ); +}); + +test('progress reports activity while idle and stops its timer on completion or failure', () => { + for (const outcome of ['done', 'failed']) { + const result = spawnSync( + process.execPath, + [ + '-e', + `const { startProgress } = require(process.argv[1]); + const finish = startProgress('Fixture', () => '; 2.0 MiB streamed to Docker', 10); + setTimeout(() => finish(process.argv[2]), 60);`, + path.join(__dirname, 'initialize-volumes.cjs'), + outcome, + ], + { encoding: 'utf8', timeout: 10000 }, + ); + assert.equal(result.status, 0, result.stderr || result.error?.message); + assert.match(result.stdout, /Fixture: starting; [\d.]+s elapsed/); + assert.match(result.stdout, /Fixture: working; [\d.]+s elapsed; 2.0 MiB streamed to Docker/); + assert.match(result.stdout.trim().split('\n').at(-1), new RegExp(`Fixture: ${outcome};`)); + } }); test('explicit archives restore dangling and directory links without traversing them', (t) => { @@ -118,6 +163,15 @@ function dockerFixture(t) { fs.writeFileSync(path.join(source, 'local-edit'), 'uncommitted'); fs.symlinkSync('local-edit', path.join(source, 'source-link')); fs.mkdirSync(path.join(source, '.worktrees/topic/gitnexus-shared'), { recursive: true }); + fs.mkdirSync(path.join(source, '.worktrees/topic/.gitnexus/parse-cache'), { recursive: true }); + fs.writeFileSync( + path.join(source, '.worktrees/topic/.gitnexus/parse-cache/chunk.v8'), + 'regenerable', + ); + fs.writeFileSync( + path.join(source, '.worktrees/topic/.gitnexus/lbug'), + 'keep graph and embeddings', + ); fs.symlinkSync( '/missing/Linux/dependencies', path.join(source, '.worktrees/topic/gitnexus-shared/node_modules'), @@ -150,7 +204,10 @@ switch(args[0]) { const dest = map(args[2].split(':').slice(1).join(':')); if (args[1] === '-') { if (process.env.IMPORT_FAIL_COPY) process.exit(3); - run('tar', ['-xf', '-', '-C', dest], fs.readFileSync(0)); + const input = fs.readFileSync(0); + if (process.env.IMPORT_SLOW_COPY && args[2].endsWith(':/seed')) + setTimeout(() => run('tar', ['-xf', '-', '-C', dest], input), 5200); + else run('tar', ['-xf', '-', '-C', dest], input); } else fs.copyFileSync(args[1], dest); break; } @@ -193,6 +250,14 @@ test('host import is one-time, uses only native volumes, preserves edits and cle assert.equal(first.status, 0, first.stderr); assert.equal(fs.readFileSync(path.join(state, 'workspace/local-edit'), 'utf8'), 'uncommitted'); assert.equal(fs.readlinkSync(path.join(state, 'workspace/source-link')), 'local-edit'); + assert.equal( + fs.existsSync(path.join(state, 'workspace/.worktrees/topic/.gitnexus/parse-cache')), + false, + ); + assert.equal( + fs.readFileSync(path.join(state, 'workspace/.worktrees/topic/.gitnexus/lbug'), 'utf8'), + 'keep graph and embeddings', + ); assert.equal( fs.existsSync(path.join(state, 'workspace/.worktrees/topic/gitnexus-shared/node_modules')), false, @@ -209,6 +274,22 @@ test('host import is one-time, uses only native volumes, preserves edits and cle const commands = fs.readFileSync(path.join(state, 'commands'), 'utf8'); assert.doesNotMatch(commands, /type=bind|--privileged|SYS_ADMIN/); assert.equal(commands.split('\n').filter((line) => line.startsWith('["cp","-"')).length, 3); + assert.match(first.stdout, /Archive to \/seed: done; .*entries queued.*MiB streamed to Docker/); + assert.match(first.stdout, /Copying into the source volume and normalizing Git: starting/); + assert.match(first.stdout, /\[workspace\] Ready/); + assert.match(first.stdout, /Finalizing \/host: done/); + assert.match(first.stdout, /Removing temporary import helper: done/); +}); + +test('archive progress continues while Docker finishes a slow copy', (t) => { + const { run } = dockerFixture(t); + const result = run('source', { IMPORT_SLOW_COPY: '1' }); + assert.equal(result.status, 0, result.stderr); + assert.match( + result.stdout, + /Archive to \/seed: working; .*entries queued.*MiB streamed to Docker/, + ); + assert.match(result.stdout, /Archive to \/seed: done/); }); test('native profile imports config without copying or requiring a host checkout', (t) => { @@ -224,6 +305,8 @@ test('a failed copy aborts initialization and never marks the source ready', (t) const { state, run } = dockerFixture(t); const result = run('source', { IMPORT_FAIL_COPY: '1' }); assert.notEqual(result.status, 0); + assert.match(result.stdout, /Archive to \/seed: failed/); + assert.doesNotMatch(result.stdout, /Archive to \/seed: done/); assert.equal(fs.existsSync(path.join(state, 'workspace/.git/devcontainer-source-volume')), false); assert.equal(fs.existsSync(path.join(state, 'active')), false); }); diff --git a/.devcontainer/seed-workspace.cjs b/.devcontainer/seed-workspace.cjs index 8da394b95..a090d0529 100644 --- a/.devcontainer/seed-workspace.cjs +++ b/.devcontainer/seed-workspace.cjs @@ -19,6 +19,13 @@ function isWorkspaceExcluded(relative) { return ( parts.includes('node_modules') || parts.includes('.venv') || + // Parser caches are regenerable and can dwarf the checkout across worktrees. + // Keep the graph, embeddings, metadata and runner in the same .gitnexus dir. + parts.some( + (part, i) => + part === '.gitnexus' && + ['parse-cache', 'parse-cache.json', 'parsedfile-cache'].includes(parts[i + 1]), + ) || parts.some((part, i) => part === '.husky' && parts[i + 1] === '_') ); } diff --git a/.devcontainer/seed-workspace.test.cjs b/.devcontainer/seed-workspace.test.cjs index b08a14f5c..d3cb43290 100644 --- a/.devcontainer/seed-workspace.test.cjs +++ b/.devcontainer/seed-workspace.test.cjs @@ -88,6 +88,13 @@ test('seed preserves Git state, local files, indexes, nested worktrees and symli fs.writeFileSync(path.join(source, 'executable'), '#!/bin/sh\n', { mode: 0o755 }); for (const file of [ '.gitnexus/index-sample', + '.gitnexus/parse-cache/chunk.v8', + '.gitnexus/parse-cache.json', + '.gitnexus/parsedfile-cache/chunk.v8', + '.worktrees/topic/.gitnexus/parse-cache/chunk.v8', + '.worktrees/topic/.gitnexus/parsedfile-cache/chunk.v8', + '.worktrees/topic/.gitnexus/lbug', + '.claude/worktrees/topic/.gitnexus/parse-cache/chunk.v8', '.worktrees/topic/local-edit', '.git/worktrees/topic/gitdir', 'eval/.venv/bin/python', @@ -133,6 +140,7 @@ test('seed preserves Git state, local files, indexes, nested worktrees and symli 'ignored-local', 'untracked', '.gitnexus/index-sample', + '.worktrees/topic/.gitnexus/lbug', '.worktrees/topic/local-edit', '.git/worktrees/topic/gitdir', '.husky/pre-commit', @@ -150,12 +158,20 @@ test('seed preserves Git state, local files, indexes, nested worktrees and symli assert.equal(fs.existsSync(path.join(target, '.husky/_')), false); assert.equal(fs.existsSync(path.join(target, 'eval/.venv')), false); for (const folder of [ + '.gitnexus/parse-cache', + '.gitnexus/parse-cache.json', + '.gitnexus/parsedfile-cache', + '.worktrees/topic/.gitnexus/parse-cache', + '.worktrees/topic/.gitnexus/parsedfile-cache', + '.claude/worktrees/topic/.gitnexus/parse-cache', '.worktrees/topic/gitnexus-shared/node_modules', '.worktrees/topic/eval/.venv', '.worktrees/topic/.husky/_', '.claude/worktrees/topic/node_modules', - ]) + ]) { assert.equal(fs.existsSync(path.join(target, folder)), false, folder); + assert.equal(fs.existsSync(path.join(source, folder)), true, `host keeps ${folder}`); + } assert.equal( fs.readFileSync(path.join(target, '.claude/worktrees/topic/local-edit'), 'utf8'), 'preserve me\n',