From 30093b2f59d6754af3330e1ae14eefa1c40b75b1 Mon Sep 17 00:00:00 2001 From: zwxxb Date: Tue, 21 Jul 2026 17:14:08 +0200 Subject: [PATCH 1/4] fix(move): provision move-flow on demand at analyze time Replace the npm-postinstall installer (scripts/install-move-flow.cjs) with analyze-time provisioning: resolve $MOVE_FLOW, then a verified user cache under ~/.gitnexus/tools/move-flow, then $PATH, then a one-time download of the pinned release verified against SHA256SUMS. Installs are serialized across processes via a lease/heartbeat lock and staged into place atomically. - Gate the client probe on the pinned release major (prerelease suffixes accepted); drop the legacy vendor/move-flow bundled path. - Honor GITNEXUS_SKIP_MOVE_FLOW and GITNEXUS_SKIP_OPTIONAL_GRAMMARS; do not retry a failed install within the same process. - Consolidate node-table DDL, CSV headers, row encoding, and COPY column lists into lbug/node-table-layout.ts (replaces move-columns.ts); align boolean CSV encoding with the incremental CREATE path. - CI: cache ~/.gitnexus/tools/move-flow keyed to the pinned version; require provisioning on the shard that owns the live Move suite. - Docker: install unzip for on-demand extraction; the image no longer ships a move-flow binary. --- .github/workflows/ci-tests.yml | 24 +- Dockerfile.cli | 5 +- gitnexus/package.json | 2 +- gitnexus/scripts/cross-platform-tests.ts | 6 + gitnexus/scripts/install-move-flow.cjs | 371 ---------- gitnexus/src/cli/analyze.ts | 13 - gitnexus/src/cli/move-availability.ts | 14 - gitnexus/src/core/ingestion/pipeline.ts | 3 +- gitnexus/src/core/lbug/csv-generator.ts | 189 ++--- gitnexus/src/core/lbug/lbug-adapter.ts | 27 +- gitnexus/src/core/lbug/move-columns.ts | 92 --- gitnexus/src/core/lbug/node-table-layout.ts | 168 +++++ gitnexus/src/core/lbug/schema.ts | 125 +--- gitnexus/src/core/move/install.ts | 663 ++++++++++++++++++ gitnexus/src/core/move/mcp-client.ts | 48 +- gitnexus/src/core/move/provision.ts | 88 +++ gitnexus/src/core/move/release.ts | 5 + gitnexus/src/core/run-analyze.ts | 4 +- gitnexus/test/integration/move-live.test.ts | 19 +- .../move-mixed-language-roundtrip.test.ts | 115 +++ .../test/unit/move/install-move-flow.test.ts | 266 +++++-- gitnexus/test/unit/move/mcp-client.test.ts | 32 +- gitnexus/test/unit/move/provision.test.ts | 107 +++ gitnexus/test/unit/node-table-layout.test.ts | 78 +++ gitnexus/vitest.config.ts | 2 + 25 files changed, 1580 insertions(+), 886 deletions(-) delete mode 100644 gitnexus/scripts/install-move-flow.cjs delete mode 100644 gitnexus/src/cli/move-availability.ts delete mode 100644 gitnexus/src/core/lbug/move-columns.ts create mode 100644 gitnexus/src/core/lbug/node-table-layout.ts create mode 100644 gitnexus/src/core/move/install.ts create mode 100644 gitnexus/src/core/move/provision.ts create mode 100644 gitnexus/src/core/move/release.ts create mode 100644 gitnexus/test/integration/move-mixed-language-roundtrip.test.ts create mode 100644 gitnexus/test/unit/move/provision.test.ts create mode 100644 gitnexus/test/unit/node-table-layout.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 0e610edd3..87a7f229a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -26,6 +26,9 @@ jobs: # FTS-dependent lbug integration suites are guaranteed to run in CI. env: GITNEXUS_REQUIRE_FTS: '1' + # The live Move suite provisions the pinned release on the shard that + # owns it and fails loudly if the compiler path is unavailable. + GITNEXUS_REQUIRE_MOVE_FLOW: '1' steps: # persist-credentials: false — runs tests + uploads a blob artifact; the # default-persisted token must not be capturable through it (zizmor @@ -34,17 +37,16 @@ jobs: with: persist-credentials: false - # Cache the move-flow binary the postinstall probe downloads into - # vendor/move-flow//. On cache hit, the probe is idempotent - # (skips the download); on miss, it downloads + verifies a pinned - # aptos-labs/aptos-ai release and soft-fails if anything goes wrong (the suite stays green; - # the live Move test simply skips when the binary is absent). Keep the - # cache key tied to the MOVE_FLOW_VERSION constant in - # gitnexus/scripts/install-move-flow.cjs. + # Cache the on-demand move-flow install so the shard owning the live Move + # suite (GITNEXUS_REQUIRE_MOVE_FLOW=1 hard-fails on provisioning errors) + # doesn't re-download the release every run and doesn't hard-fail on a + # transient GitHub Releases outage. A restored cache is still verified + # (metadata + binary sha256 + version probe) before use. Keep the key + # tied to MOVE_FLOW_RELEASE.version in gitnexus/src/core/move/release.ts. - name: Cache move-flow binary - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: - path: gitnexus/vendor/move-flow + path: ~/.gitnexus/tools/move-flow key: ${{ runner.os }}-move-flow-v2.0.0 - uses: ./.github/actions/setup-gitnexus @@ -209,8 +211,8 @@ jobs: timeout-minutes: 25 # Same guarantee on the platform-sensitive runners: FTS-dependent suites in # the cross-platform subset must run, not silently skip. Move ingestion isn't - # exercised by this subset either, so GITNEXUS_SKIP_MOVE_FLOW below drops the - # postinstall move-flow probe as pure wasted bandwidth here. + # exercised by this subset either, so GITNEXUS_SKIP_MOVE_FLOW prevents any + # fixture-driven analyze smoke from provisioning it as wasted bandwidth here. # # GITNEXUS_E2E_CLI=dist: the e2e suites spawn the CLI ~50 times; each spawn via # `node --import tsx src/cli/index.ts` re-transpiles the whole CLI, and Windows diff --git a/Dockerfile.cli b/Dockerfile.cli index 633d23f5d..fabef5ca1 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -51,8 +51,9 @@ RUN npm run postinstall --prefix gitnexus # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -# curl for the healthcheck; git for cloning; ca-certificates for TLS verification. -RUN apt-get update && apt-get install -y --no-install-recommends curl git ca-certificates && rm -rf /var/lib/apt/lists/* \ +# curl for the healthcheck; git for cloning; unzip for on-demand Move Flow; +# ca-certificates for TLS verification. +RUN apt-get update && apt-get install -y --no-install-recommends curl git unzip ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/local/lib/node_modules/npm \ && rm -rf /usr/local/lib/node_modules/corepack \ && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack diff --git a/gitnexus/package.json b/gitnexus/package.json index d2870e43f..a62039186 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -54,7 +54,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", - "postinstall": "node scripts/build-tree-sitter-grammars.cjs && node scripts/install-move-flow.cjs", + "postinstall": "node scripts/build-tree-sitter-grammars.cjs", "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js", diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 5d6b177f5..2af0be467 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -69,6 +69,9 @@ const PLATFORM_LOGIC = [ // array form. Runs on every platform (the ubuntu suite covers Linux; this // registration adds windows + macos). 'test/unit/embedding-install-arg-delivery.test.ts', + // Lazy MoveFlow install coordinates, PowerShell invocation data, archive + // streaming, and filesystem leases all vary across operating systems. + 'test/unit/move/install-move-flow.test.ts', // Structural FTS-extension classifier against REAL binaries (#2374): on this // matrix `process.execPath` / `lbugjs.node` are a real PE (windows) and Mach-O // (macos), so the header parsing is proven on genuine binaries, not synthetic @@ -95,6 +98,9 @@ const LBUG_NATIVE = [ 'test/integration/lbug-orphan-sidecar-recovery.test.ts', 'test/integration/lbug-readonly-init.test.ts', 'test/integration/lbug-non-ascii-path.test.ts', + // Shared Move/non-Move rows must retain their schema defaults through the + // real native database on every supported desktop platform. + 'test/integration/move-mixed-language-roundtrip.test.ts', // Cross-repo trace e2e: builds two real lbug indexes + a real bridge and // opens them through the pool adapter (native addon + bridge file locking). // Windows is skipped in-file (describeReopen) due to the bridge reopen lock. diff --git a/gitnexus/scripts/install-move-flow.cjs b/gitnexus/scripts/install-move-flow.cjs deleted file mode 100644 index 24d83f82f..000000000 --- a/gitnexus/scripts/install-move-flow.cjs +++ /dev/null @@ -1,371 +0,0 @@ -#!/usr/bin/env node -/** - * Optional native dependency probe: download a pinned `move-flow` release - * into `gitnexus/vendor/move-flow//`. - * - * Same shape as `build-tree-sitter-grammars.cjs`: an opt-out env var, a soft-fail - * contract that always exits 0 (the gitnexus install must succeed even when - * the binary can't be provisioned), idempotency (skip when the on-disk - * version already matches), and an offline platform detect that warns and - * exits cleanly on unsupported targets. - * - * The runtime side (`tryCreateMoveFlowClient`) resolves the binary in this - * order: $MOVE_FLOW → bundled `vendor/move-flow//move-flow[.exe]` - * → $PATH. When this probe fails, the Move ingest phase no-ops for non-Move - * repos and the analyze CLI emits a one-line stderr notice for Move repos. - * - * Supported env: - * GITNEXUS_SKIP_MOVE_FLOW=1 skip this probe - * GITNEXUS_MOVE_FLOW_VERSION=x.y.z override the pinned version - * GITNEXUS_MOVE_FLOW_REPO=owner/repo override the release repository - * GITNEXUS_MOVE_FLOW_TAG=tag override the release tag - * GITNEXUS_MOVE_FLOW_COMPAT=1 force Linux compat artifact - */ - -'use strict'; - -const fs = require('node:fs'); -const path = require('node:path'); -const https = require('node:https'); -const crypto = require('node:crypto'); -const { execFileSync } = require('node:child_process'); -const os = require('node:os'); -const { pipeline } = require('node:stream'); - -/** Pinned move-flow release from https://github.com/aptos-labs/aptos-ai. */ -const MOVE_FLOW_VERSION = process.env.GITNEXUS_MOVE_FLOW_VERSION || '2.0.0'; -const RELEASE_REPO = process.env.GITNEXUS_MOVE_FLOW_REPO || 'aptos-labs/aptos-ai'; -const RELEASE_TAG = process.env.GITNEXUS_MOVE_FLOW_TAG || `move-flow-v${MOVE_FLOW_VERSION}`; -const RELEASE_BASE = `https://github.com/${RELEASE_REPO}/releases/download/${RELEASE_TAG}`; -const BIN_NAME = process.platform === 'win32' ? 'move-flow.exe' : 'move-flow'; - -const SKIP_FLAGS = ['GITNEXUS_SKIP_MOVE_FLOW', 'GITNEXUS_SKIP_OPTIONAL_GRAMMARS']; - -const vendorRoot = path.join(__dirname, '..', 'vendor', 'move-flow'); - -function platformKey() { - const { platform, arch } = process; - if (platform === 'linux' && arch === 'x64') return 'linux-x64'; - if (platform === 'linux' && arch === 'arm64') return 'linux-arm64'; - if (platform === 'darwin' && arch === 'arm64') return 'darwin-arm64'; - if (platform === 'darwin' && arch === 'x64') return 'darwin-x64'; - if (platform === 'win32' && arch === 'x64') return 'win32-x64'; - return null; -} - -function targetBinaryPath(key) { - const dir = path.join(vendorRoot, key); - return { dir, file: path.join(dir, BIN_NAME) }; -} - -function versionMatches(file) { - try { - const out = execFileSync(file, ['--version'], { - encoding: 'utf8', - timeout: 5000, - stdio: ['ignore', 'pipe', 'ignore'], - }); - return out.includes(MOVE_FLOW_VERSION); - } catch { - return false; - } -} - -function linuxNeedsCompatBuild() { - if (process.platform !== 'linux') return false; - if (process.env.GITNEXUS_MOVE_FLOW_COMPAT === '1') return true; - if (process.arch === 'x64') { - try { - const cpuinfo = fs.readFileSync('/proc/cpuinfo', 'utf8'); - if (!/(^|\s)avx2(\s|$)/m.test(cpuinfo)) return true; - } catch { - return true; - } - } - try { - const out = execFileSync('ldd', ['--version'], { - encoding: 'utf8', - timeout: 3000, - stdio: ['ignore', 'pipe', 'ignore'], - }); - const match = /(\d+)\.(\d+)/.exec(out.split('\n')[0] ?? ''); - if (!match) return false; - const major = Number(match[1]); - const minor = Number(match[2]); - return major < 2 || (major === 2 && minor < 34); - } catch { - return false; - } -} - -function releaseTargetForPlatform(key) { - switch (key) { - case 'darwin-arm64': - return 'aarch64-apple-darwin'; - case 'darwin-x64': - return 'x86_64-apple-darwin'; - case 'linux-arm64': - return `aarch64-unknown-linux-gnu${linuxNeedsCompatBuild() ? '-compat' : ''}`; - case 'linux-x64': - return `x86_64-unknown-linux-gnu${linuxNeedsCompatBuild() ? '-compat' : ''}`; - case 'win32-x64': - return 'x86_64-pc-windows-msvc'; - default: - return null; - } -} - -function downloadToFile(url, dest, get = https.get) { - return new Promise((resolve, reject) => { - const tmp = `${dest}.partial`; - let settled = false; - - const fail = (err) => { - if (settled) return; - settled = true; - try { - fs.rmSync(tmp, { force: true }); - } catch { - /* the caller's temp-directory cleanup gets a second chance */ - } - reject(err); - }; - - const followRedirect = (target, hops) => { - if (hops > 5) { - fail(new Error('too many redirects')); - return; - } - const req = get(target, (res) => { - const status = res.statusCode || 0; - if (status >= 300 && status < 400 && res.headers.location) { - res.on('error', fail); - res.resume(); - followRedirect(new URL(res.headers.location, target).toString(), hops + 1); - return; - } - if (status !== 200) { - res.on('error', fail); - res.resume(); - fail(new Error(`HTTP ${status} for ${target}`)); - return; - } - - // pipeline owns both streams and reports source (mid-download) and - // destination errors through one callback instead of allowing an - // unhandled stream 'error' event to escape the postinstall soft-fail. - pipeline(res, fs.createWriteStream(tmp), (err) => { - if (settled) return; - if (err) { - fail(err); - return; - } - try { - fs.renameSync(tmp, dest); - settled = true; - resolve(); - } catch (renameErr) { - fail(renameErr); - } - }); - }); - req.on('error', fail); - }; - followRedirect(url, 0); - }); -} - -function sha256(file) { - const hash = crypto.createHash('sha256'); - hash.update(fs.readFileSync(file)); - return hash.digest('hex'); -} - -function powershellExpandArchiveInvocation(archive, dest) { - const archiveEnv = 'GITNEXUS_MOVE_FLOW_ARCHIVE_PATH'; - const destinationEnv = 'GITNEXUS_MOVE_FLOW_DESTINATION_PATH'; - return { - args: [ - '-NoProfile', - '-NonInteractive', - '-Command', - `Expand-Archive -LiteralPath $env:${archiveEnv} -DestinationPath $env:${destinationEnv} -Force`, - ], - env: { - ...process.env, - [archiveEnv]: archive, - [destinationEnv]: dest, - }, - }; -} - -function extractZip(archive, dest) { - fs.mkdirSync(dest, { recursive: true }); - try { - if (process.platform === 'win32') { - const ps = process.env.ComSpec ? 'powershell.exe' : 'powershell'; - const invocation = powershellExpandArchiveInvocation(archive, dest); - execFileSync(ps, invocation.args, { - stdio: 'ignore', - timeout: 30000, - env: invocation.env, - }); - return; - } - execFileSync('unzip', ['-q', archive, '-d', dest], { stdio: 'ignore', timeout: 30000 }); - } catch (err) { - throw new Error( - `could not extract ${path.basename(archive)} (${err instanceof Error ? err.message : err}). ` + - 'Install unzip, or set MOVE_FLOW to an existing move-flow binary.', - ); - } -} - -function findExtractedBinary(root) { - const stack = [root]; - const names = new Set([BIN_NAME, 'move-flow']); - while (stack.length > 0) { - const current = stack.pop(); - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const full = path.join(current, entry.name); - if (entry.isDirectory()) { - stack.push(full); - } else if (names.has(entry.name)) { - return full; - } - } - } - return null; -} - -function expectedSha(sumsText, assetName) { - for (const raw of sumsText.split('\n')) { - const line = raw.trim(); - if (!line) continue; - // Format: " " (BSD-style "SHA256 (file) = " also tolerated). - const m = /^([0-9a-f]{64})[ \t*]+(\S.*)$/i.exec(line); - if (m && m[2] === assetName) return m[1].toLowerCase(); - const bsd = /^SHA256\s*\((.*)\)\s*=\s*([0-9a-f]{64})$/i.exec(line); - if (bsd && bsd[1] === assetName) return bsd[2].toLowerCase(); - } - return null; -} - -function verifyArchiveChecksum(sumsText, assetName, archive) { - const expected = expectedSha(sumsText, assetName); - if (!expected) return { status: 'missing' }; - - const actual = sha256(archive); - if (actual !== expected) return { status: 'mismatch', expected, actual }; - return { status: 'match', expected, actual }; -} - -async function main() { - for (const flag of SKIP_FLAGS) { - if (process.env[flag] === '1') { - console.warn(`[move-flow] Skipping install (${flag}=1).`); - process.exit(0); - } - } - - const key = platformKey(); - if (!key) { - console.warn( - `[move-flow] Unsupported platform ${process.platform}-${process.arch} — skipping. ` + - 'Set $MOVE_FLOW to provide a binary, or set GITNEXUS_SKIP_MOVE_FLOW=1 to silence this notice.', - ); - process.exit(0); - } - - const releaseTarget = releaseTargetForPlatform(key); - - const { dir, file } = targetBinaryPath(key); - - if (fs.existsSync(file) && versionMatches(file)) { - // Idempotent: already provisioned at the right version. - process.exit(0); - } - - const assetName = `${RELEASE_TAG}-${releaseTarget}.zip`; - const sumsName = 'SHA256SUMS'; - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'move-flow-')); - const tmpArchive = path.join(tmpDir, assetName); - const extractDir = path.join(tmpDir, 'extract'); - const tmpSums = path.join(tmpDir, sumsName); - - try { - await downloadToFile(`${RELEASE_BASE}/${sumsName}`, tmpSums); - await downloadToFile(`${RELEASE_BASE}/${assetName}`, tmpArchive); - - const verification = verifyArchiveChecksum( - fs.readFileSync(tmpSums, 'utf8'), - assetName, - tmpArchive, - ); - if (verification.status === 'missing') { - console.warn( - `[move-flow] ${sumsName} does not list ${assetName} — refusing to install. ` + - 'Move ingestion will be unavailable. Non-Move functionality is unaffected.', - ); - process.exit(0); - } - - if (verification.status === 'mismatch') { - console.warn( - `[move-flow] Checksum mismatch for ${assetName} (expected ${verification.expected}, got ${verification.actual}) — refusing to install. ` + - 'Move ingestion will be unavailable. Non-Move functionality is unaffected.', - ); - process.exit(0); - } - - extractZip(tmpArchive, extractDir); - const extracted = findExtractedBinary(extractDir); - if (!extracted) { - console.warn( - `[move-flow] ${assetName} did not contain ${BIN_NAME} — refusing to install. ` + - 'Move ingestion will be unavailable. Non-Move functionality is unaffected.', - ); - process.exit(0); - } - - fs.mkdirSync(dir, { recursive: true }); - fs.copyFileSync(extracted, file); - try { - fs.chmodSync(file, 0o755); - } catch { - /* no-op on Windows */ - } - if (!versionMatches(file)) { - fs.rmSync(file, { force: true }); - console.warn( - `[move-flow] Installed binary did not report version ${MOVE_FLOW_VERSION} — refusing to keep it. ` + - 'Move ingestion will be unavailable. Non-Move functionality is unaffected.', - ); - } - } catch (err) { - console.warn(`[move-flow] Download failed: ${err instanceof Error ? err.message : err}`); - console.warn( - '[move-flow] Move ingestion will be unavailable. Set $MOVE_FLOW to a local binary, ' + - 'or set GITNEXUS_SKIP_MOVE_FLOW=1 to silence this notice. Non-Move functionality is unaffected.', - ); - process.exit(0); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -} - -module.exports = { - downloadToFile, - expectedSha, - sha256, - verifyArchiveChecksum, - powershellExpandArchiveInvocation, -}; - -if (require.main === module) { - main().catch((err) => { - // Never hard-fail the gitnexus install. - console.warn(`[move-flow] Unexpected error during install probe: ${err?.message ?? err}`); - process.exit(0); - }); -} diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index f373e0c14..44d0182ed 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -40,11 +40,9 @@ import { GitNexusRcError, } from './analyze-config.js'; import { runFullAnalysis } from '../core/run-analyze.js'; -import { repoHasMove } from '../core/move/discovery.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; import { warnMissingOptionalGrammars, getOptionalGrammarExtensions } from './optional-grammars.js'; -import { warnIfMoveUnavailable } from './move-availability.js'; import { glob } from 'glob'; import fs from 'fs/promises'; import { cliError } from './cli-message.js'; @@ -1222,17 +1220,6 @@ const analyzeCommandImpl = async ( // Best-effort warning \u2014 never block analyze on the precheck. } - // Move ingestion is compiler-first via move-flow; warn once if the repo - // has Move sources but no usable binary is reachable. Uses the shared - // `repoHasMove` helper so the precheck keys off the same Move.toml signal - // that the ingestion phase actually uses (a repo with loose `.move` files - // but no Move.toml would warn but ingest nothing). - try { - warnIfMoveUnavailable({ context: 'analyze', repoHasMove: await repoHasMove(repoPath) }); - } catch { - // Best-effort \u2014 never block analyze on the precheck. - } - // KuzuDB migration cleanup is handled by runFullAnalysis internally. // Note: --skills is handled after runFullAnalysis using the returned pipelineResult. diff --git a/gitnexus/src/cli/move-availability.ts b/gitnexus/src/cli/move-availability.ts deleted file mode 100644 index 0ce2823e3..000000000 --- a/gitnexus/src/cli/move-availability.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** Warn once if a repo has Move sources but no usable move-flow binary is reachable. */ - -import { tryCreateMoveFlowClient } from '../core/move/mcp-client.js'; -import { cliWarn } from './cli-message.js'; - -export function warnIfMoveUnavailable(opts: { repoHasMove: boolean; context?: string }): void { - if (!opts.repoHasMove) return; - if (tryCreateMoveFlowClient()) return; - const ctx = opts.context ? ` [${opts.context}]` : ''; - cliWarn( - `GitNexus${ctx}: move-flow is unavailable — .move files will not be indexed. Reinstall without GITNEXUS_SKIP_MOVE_FLOW=1, or set MOVE_FLOW to a move-flow binary from aptos-labs/aptos-ai.`, - { binary: 'move-flow', context: opts.context }, - ); -} diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index c4e074ab7..15d966057 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -261,11 +261,12 @@ export interface PipelineOptions { * options combination. */ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { + const { standaloneIngestPhase = emptyStandaloneIngestPhase } = options ?? {}; return ( new PhaseRegistry() .register(scanPhase) .register(structurePhase) - .register(options?.standaloneIngestPhase ?? emptyStandaloneIngestPhase) + .register(standaloneIngestPhase) .register(markdownPhase) .register(cobolPhase) .register(parsePhase) diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 682a755f0..540d88673 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -24,13 +24,12 @@ import { SYMBOL_NODE_LABELS } from '../ingestion/utils/symbol-labels.js'; import { applyCjkSegmentationIfEnabled } from '../search/cjk-segmentation.js'; import { CODE_ELEMENT_COLUMNS, - MOVE_CONST_COLUMNS, - MOVE_ENUM_VARIANT_COLUMNS, - MOVE_FUNCTION_COLUMNS, - MOVE_MODULE_COLUMNS, - MOVE_STRUCT_LIKE_COLUMNS, + getNodeTableCsvHeader, + getNodeTableLayout, + hasNodeTableLayout, MULTI_LANG_BASE_COLUMNS, -} from './move-columns.js'; + type LayoutTableName, +} from './node-table-layout.js'; /** * Deterministic output ordering — optional (out-of-core / windowed-resolve @@ -141,8 +140,10 @@ export const escapeCSVStringArray = (value: unknown): string => { return escapeCSVField(`[${encoded.join(',')}]`); }; +// Truthy coercion, matching the incremental CREATE path (`!!properties.isExported` +// in lbug-adapter) so bulk and incremental loads classify the same values alike. export const escapeCSVBoolean = (value: unknown): string => { - return value === true ? 'true' : 'false'; + return value ? 'true' : 'false'; }; // ============================================================================ @@ -241,6 +242,44 @@ export const normalizeFtsText = (text: string): string => text.replace(/[\r\n\t] const formatFtsDescription = (description: string): string => normalizeFtsText(applyCjkSegmentationIfEnabled(description)); +/** + * Encode a row from the same ordered layout that owns its DDL, CSV header, + * and COPY column list. These tables are shared across languages, so missing + * optional properties deliberately serialize to typed defaults. + */ +export const buildLayoutNodeRow = ( + table: LayoutTableName, + node: GraphNode, + content: string, +): string => { + const layout = getNodeTableLayout(table); + + return layout.columns + .map((spec) => { + const value = node.properties[spec.name] ?? spec.defaultValue; + switch (spec.csvEncoding) { + case 'node-id': + return escapeCSVField(node.id); + case 'number': + return escapeCSVNumber( + typeof value === 'number' ? value : undefined, + typeof spec.defaultValue === 'number' ? spec.defaultValue : -1, + ); + case 'boolean': + return escapeCSVBoolean(value); + case 'string-array': + return escapeCSVStringArray(value); + case 'content': + return escapeCSVField(content); + case 'description': + return escapeCSVField(formatFtsDescription(String(value ?? ''))); + case 'string': + return escapeCSVField(String(value ?? '')); + } + }) + .join(','); +}; + // Labels that get exact source-span content (no ±2 window). Single source of // truth in `symbol-labels.ts` — see there for why the exactness depends on the // 0-based line invariant. Kept as a named alias to read intent at the use site. @@ -454,7 +493,7 @@ export const streamAllCSVsToDisk = async ( const codeElementHeader = CODE_ELEMENT_COLUMNS.join(','); const functionWriter = new BufferedCSVWriter( path.join(csvDir, 'function.csv'), - MOVE_FUNCTION_COLUMNS.join(','), + getNodeTableCsvHeader('Function'), ); const classWriter = new BufferedCSVWriter(path.join(csvDir, 'class.csv'), codeElementHeader); const interfaceWriter = new BufferedCSVWriter( @@ -504,10 +543,6 @@ export const streamAllCSVsToDisk = async ( // Multi-language node types share the same CSV shape (no isExported column) const multiLangHeader = MULTI_LANG_BASE_COLUMNS.join(','); - const moveStructLikeHeader = MOVE_STRUCT_LIKE_COLUMNS.join(','); - const moveConstHeader = MOVE_CONST_COLUMNS.join(','); - const moveEnumVariantHeader = MOVE_ENUM_VARIANT_COLUMNS.join(','); - const moveModuleHeader = MOVE_MODULE_COLUMNS.join(','); const MULTI_LANG_TYPES = [ 'Struct', 'Enum', @@ -539,15 +574,9 @@ export const streamAllCSVsToDisk = async ( path.join(csvDir, `${t.toLowerCase()}.csv`), t === 'Property' ? propertyHeader - : t === 'Struct' || t === 'Enum' - ? moveStructLikeHeader - : t === 'EnumVariant' - ? moveEnumVariantHeader - : t === 'Const' - ? moveConstHeader - : t === 'Module' - ? moveModuleHeader - : multiLangHeader, + : hasNodeTableLayout(t) + ? getNodeTableCsvHeader(t) + : multiLangHeader, ), ); } @@ -704,119 +733,39 @@ export const streamAllCSVsToDisk = async ( const writer = codeWriterMap[node.label]; if (writer) { const content = await extractContent(node, contentCache); - const baseFields = [ - escapeCSVField(node.id), - escapeCSVField(node.properties.name || ''), - escapeCSVField(node.properties.filePath || ''), - escapeCSVNumber(node.properties.startLine, -1), - escapeCSVNumber(node.properties.endLine, -1), - node.properties.isExported ? 'true' : 'false', - escapeCSVField(content), - escapeCSVField(formatFtsDescription(node.properties.description || '')), - ]; if (node.label === 'Function') { + pending = writer.addRow(buildLayoutNodeRow('Function', node, content)); + } else { pending = writer.addRow( [ - ...baseFields, - escapeCSVField(node.properties.language || ''), - escapeCSVField(node.properties.qualifiedName || ''), - escapeCSVField(node.properties.moduleQualifiedName || ''), - escapeCSVField(node.properties.visibility || ''), - escapeCSVField(node.properties.visibilityModifier || ''), - escapeCSVBoolean(node.properties.isEntry), - escapeCSVBoolean(node.properties.isView), - escapeCSVBoolean(node.properties.isInline), - escapeCSVBoolean(node.properties.isNative), - escapeCSVNumber(node.properties.parameterCount, 0), - escapeCSVField(node.properties.returnType || ''), - escapeCSVStringArray(node.properties.acquires), - escapeCSVStringArray(node.properties.usedTypes), - escapeCSVStringArray(node.properties.attributes), - escapeCSVField(String(node.properties.attributesJson ?? '')), - escapeCSVField(String(node.properties.typeParamsJson ?? '')), - escapeCSVField(String(node.properties.locationFidelity ?? '')), + escapeCSVField(node.id), + escapeCSVField(node.properties.name || ''), + escapeCSVField(node.properties.filePath || ''), + escapeCSVNumber(node.properties.startLine, -1), + escapeCSVNumber(node.properties.endLine, -1), + node.properties.isExported ? 'true' : 'false', + escapeCSVField(content), + escapeCSVField(formatFtsDescription(node.properties.description || '')), ].join(','), ); - } else { - pending = writer.addRow(baseFields.join(',')); } } else { // Multi-language node types (Struct, Impl, Trait, Macro, etc.) const mlWriter = multiLangWriters.get(node.label); if (mlWriter) { const content = await extractContent(node, contentCache); - const baseFields = [ - escapeCSVField(node.id), - escapeCSVField(node.properties.name || ''), - escapeCSVField(node.properties.filePath || ''), - escapeCSVNumber(node.properties.startLine, -1), - escapeCSVNumber(node.properties.endLine, -1), - escapeCSVField(content), - escapeCSVField(formatFtsDescription(node.properties.description || '')), - ]; - if (node.label === 'Struct' || node.label === 'Enum') { - pending = mlWriter.addRow( - [ - ...baseFields, - escapeCSVField(node.properties.language || ''), - escapeCSVField(node.properties.qualifiedName || ''), - escapeCSVField(node.properties.moduleQualifiedName || ''), - escapeCSVField(node.properties.moduleAddress || ''), - escapeCSVStringArray(node.properties.abilities), - escapeCSVBoolean(node.properties.isResource), - escapeCSVBoolean(node.properties.isEvent), - escapeCSVStringArray(node.properties.fieldList), - escapeCSVStringArray(node.properties.attributes), - escapeCSVField(String(node.properties.attributesJson ?? '')), - escapeCSVField(String(node.properties.typeParamsJson ?? '')), - escapeCSVField(node.properties.moveDeclarationKind || ''), - escapeCSVField(String(node.properties.locationFidelity ?? '')), - ].join(','), - ); - } else if (node.label === 'EnumVariant') { - pending = mlWriter.addRow( - [ - ...baseFields, - escapeCSVField(node.properties.language || ''), - escapeCSVField(node.properties.qualifiedName || ''), - escapeCSVField(String(node.properties.parentEnum || '')), - escapeCSVField(String(node.properties.moduleQualifiedName || '')), - escapeCSVField(String(node.properties.variantKind || '')), - escapeCSVField(String(node.properties.fieldsJson ?? '')), - escapeCSVStringArray(node.properties.attributes), - escapeCSVField(String(node.properties.attributesJson ?? '')), - escapeCSVField(String(node.properties.locationFidelity ?? '')), - ].join(','), - ); - } else if (node.label === 'Const') { - pending = mlWriter.addRow( - [ - ...baseFields, - escapeCSVField(node.properties.language || ''), - escapeCSVField(node.properties.qualifiedName || ''), - escapeCSVField(node.properties.moduleQualifiedName || ''), - escapeCSVField(String(node.properties.constType ?? '')), - escapeCSVField(String(node.properties.constValue ?? '')), - escapeCSVBoolean(node.properties.isErrorCode), - escapeCSVField(String(node.properties.locationFidelity ?? '')), - ].join(','), - ); - } else if (node.label === 'Module') { - pending = mlWriter.addRow( - [ - ...baseFields, - escapeCSVField(node.properties.language || ''), - escapeCSVField(node.properties.qualifiedName || ''), - escapeCSVField(node.properties.moduleAddress || ''), - escapeCSVStringArray(node.properties.attributes), - escapeCSVField(String(node.properties.attributesJson ?? '')), - escapeCSVField(String(node.properties.locationFidelity ?? '')), - ].join(','), - ); + if (hasNodeTableLayout(node.label)) { + pending = mlWriter.addRow(buildLayoutNodeRow(node.label, node, content)); } else { pending = mlWriter.addRow( [ - ...baseFields, + escapeCSVField(node.id), + escapeCSVField(node.properties.name || ''), + escapeCSVField(node.properties.filePath || ''), + escapeCSVNumber(node.properties.startLine, -1), + escapeCSVNumber(node.properties.endLine, -1), + escapeCSVField(content), + escapeCSVField(formatFtsDescription(node.properties.description || '')), ...(node.label === 'Property' ? [escapeCSVField(node.properties.declaredType || '')] : []), diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 698e35ac9..8dde4b2dc 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -23,13 +23,7 @@ import { streamAllCSVsToDisk, type StreamedCSVResult } from './csv-generator.js' import type { PdgEmitManifest } from './pdg-emit-sink.js'; import { getNodeLabel as deriveNodeLabel, type WriteStreamFactory } from './rel-pair-routing.js'; import { EMBEDDABLE_LABELS, type CachedEmbedding } from '../embeddings/types.js'; -import { - MOVE_CONST_COLUMNS, - MOVE_ENUM_VARIANT_COLUMNS, - MOVE_FUNCTION_COLUMNS, - MOVE_MODULE_COLUMNS, - MOVE_STRUCT_LIKE_COLUMNS, -} from './move-columns.js'; +import { getNodeTableColumnNames } from './node-table-layout.js'; import { extensionManager, type ExtensionEnsureOptions } from './extension-loader.js'; import { classifyDeleteAllError, @@ -1357,6 +1351,10 @@ const copyColumns = (columns: readonly string[]): string => columns.join(', '); export const getCopyQuery = (table: NodeTableName, filePath: string): string => { const t = escapeTableName(table); + const layoutColumns = getNodeTableColumnNames(table); + if (layoutColumns) { + return `COPY ${t}(${copyColumns(layoutColumns)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; + } if (table === 'File') { return `COPY ${t}(id, name, filePath, content) FROM "${filePath}" ${COPY_CSV_OPTS}`; } @@ -1390,21 +1388,6 @@ export const getCopyQuery = (table: NodeTableName, filePath: string): string => if (table === 'Property') { return `COPY ${t}(id, name, filePath, startLine, endLine, content, description, declaredType) FROM "${filePath}" ${COPY_CSV_OPTS}`; } - if (table === 'Function') { - return `COPY ${t}(${copyColumns(MOVE_FUNCTION_COLUMNS)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; - } - if (table === 'Struct' || table === 'Enum') { - return `COPY ${t}(${copyColumns(MOVE_STRUCT_LIKE_COLUMNS)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; - } - if (table === 'EnumVariant') { - return `COPY ${t}(${copyColumns(MOVE_ENUM_VARIANT_COLUMNS)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; - } - if (table === 'Const') { - return `COPY ${t}(${copyColumns(MOVE_CONST_COLUMNS)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; - } - if (table === 'Module') { - return `COPY ${t}(${copyColumns(MOVE_MODULE_COLUMNS)}) FROM "${filePath}" ${COPY_CSV_OPTS}`; - } // TypeScript/JS code element tables have isExported; multi-language tables do not if (TABLES_WITH_EXPORTED.has(table)) { return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description) FROM "${filePath}" ${COPY_CSV_OPTS}`; diff --git a/gitnexus/src/core/lbug/move-columns.ts b/gitnexus/src/core/lbug/move-columns.ts deleted file mode 100644 index e10ce4a1f..000000000 --- a/gitnexus/src/core/lbug/move-columns.ts +++ /dev/null @@ -1,92 +0,0 @@ -export const CODE_ELEMENT_COLUMNS = [ - 'id', - 'name', - 'filePath', - 'startLine', - 'endLine', - 'isExported', - 'content', - 'description', -] as const; - -export const MULTI_LANG_BASE_COLUMNS = [ - 'id', - 'name', - 'filePath', - 'startLine', - 'endLine', - 'content', - 'description', -] as const; - -export const MOVE_FUNCTION_COLUMNS = [ - ...CODE_ELEMENT_COLUMNS, - 'language', - 'qualifiedName', - 'moduleQualifiedName', - 'visibility', - 'visibilityModifier', - 'isEntry', - 'isView', - 'isInline', - 'isNative', - 'parameterCount', - 'returnType', - 'acquires', - 'usedTypes', - 'attributes', - 'attributesJson', - 'typeParamsJson', - 'locationFidelity', -] as const; - -export const MOVE_STRUCT_LIKE_COLUMNS = [ - ...MULTI_LANG_BASE_COLUMNS, - 'language', - 'qualifiedName', - 'moduleQualifiedName', - 'moduleAddress', - 'abilities', - 'isResource', - 'isEvent', - 'fieldList', - 'attributes', - 'attributesJson', - 'typeParamsJson', - 'moveDeclarationKind', - 'locationFidelity', -] as const; - -export const MOVE_ENUM_VARIANT_COLUMNS = [ - ...MULTI_LANG_BASE_COLUMNS, - 'language', - 'qualifiedName', - 'parentEnum', - 'moduleQualifiedName', - 'variantKind', - 'fieldsJson', - 'attributes', - 'attributesJson', - 'locationFidelity', -] as const; - -export const MOVE_CONST_COLUMNS = [ - ...MULTI_LANG_BASE_COLUMNS, - 'language', - 'qualifiedName', - 'moduleQualifiedName', - 'constType', - 'constValue', - 'isErrorCode', - 'locationFidelity', -] as const; - -export const MOVE_MODULE_COLUMNS = [ - ...MULTI_LANG_BASE_COLUMNS, - 'language', - 'qualifiedName', - 'moduleAddress', - 'attributes', - 'attributesJson', - 'locationFidelity', -] as const; diff --git a/gitnexus/src/core/lbug/node-table-layout.ts b/gitnexus/src/core/lbug/node-table-layout.ts new file mode 100644 index 000000000..420b4070c --- /dev/null +++ b/gitnexus/src/core/lbug/node-table-layout.ts @@ -0,0 +1,168 @@ +import type { NodeTableName } from 'gitnexus-shared'; + +export type CsvColumnEncoding = + | 'node-id' + | 'string' + | 'number' + | 'boolean' + | 'string-array' + | 'content' + | 'description'; + +export interface NodeTableColumnSpec { + name: string; + ddlType: string; + csvEncoding: CsvColumnEncoding; + defaultValue?: string | number | boolean; +} + +export interface NodeTableLayout { + table: NodeTableName; + columns: readonly NodeTableColumnSpec[]; + quoteTableName?: boolean; +} + +const column = ( + name: string, + ddlType: string, + csvEncoding: CsvColumnEncoding = 'string', + defaultValue?: string | number | boolean, +): NodeTableColumnSpec => ({ name, ddlType, csvEncoding, defaultValue }); + +const CODE_ELEMENT_BASE = [ + column('id', 'STRING', 'node-id'), + column('name', 'STRING'), + column('filePath', 'STRING'), + column('startLine', 'INT64', 'number', -1), + column('endLine', 'INT64', 'number', -1), + column('isExported', 'BOOLEAN', 'boolean', false), + column('content', 'STRING', 'content'), + column('description', 'STRING', 'description'), +] as const; + +// Same base as code elements minus isExported (multi-language tables have none). +const MULTI_LANGUAGE_BASE = CODE_ELEMENT_BASE.filter(({ name }) => name !== 'isExported'); + +export const CODE_ELEMENT_COLUMNS = CODE_ELEMENT_BASE.map(({ name }) => name); +export const MULTI_LANG_BASE_COLUMNS = MULTI_LANGUAGE_BASE.map(({ name }) => name); + +const FUNCTION_LAYOUT: NodeTableLayout = { + table: 'Function', + columns: [ + ...CODE_ELEMENT_BASE, + column('language', 'STRING'), + column('qualifiedName', 'STRING'), + column('moduleQualifiedName', 'STRING'), + column('visibility', 'STRING'), + column('visibilityModifier', 'STRING'), + column('isEntry', 'BOOLEAN', 'boolean', false), + column('isView', 'BOOLEAN', 'boolean', false), + column('isInline', 'BOOLEAN', 'boolean', false), + column('isNative', 'BOOLEAN', 'boolean', false), + column('parameterCount', 'INT32', 'number', 0), + column('returnType', 'STRING'), + column('acquires', 'STRING[]', 'string-array'), + column('usedTypes', 'STRING[]', 'string-array'), + column('attributes', 'STRING[]', 'string-array'), + column('attributesJson', 'STRING'), + column('typeParamsJson', 'STRING'), + column('locationFidelity', 'STRING'), + ], +}; + +const structLikeLayout = (table: 'Struct' | 'Enum'): NodeTableLayout => ({ + table, + quoteTableName: true, + columns: [ + ...MULTI_LANGUAGE_BASE, + column('language', 'STRING'), + column('qualifiedName', 'STRING'), + column('moduleQualifiedName', 'STRING'), + column('moduleAddress', 'STRING'), + column('abilities', 'STRING[]', 'string-array'), + column('isResource', 'BOOLEAN', 'boolean', false), + column('isEvent', 'BOOLEAN', 'boolean', false), + column('fieldList', 'STRING[]', 'string-array'), + column('attributes', 'STRING[]', 'string-array'), + column('attributesJson', 'STRING'), + column('typeParamsJson', 'STRING'), + column('moveDeclarationKind', 'STRING'), + column('locationFidelity', 'STRING'), + ], +}); + +const ENUM_VARIANT_LAYOUT: NodeTableLayout = { + table: 'EnumVariant', + quoteTableName: true, + columns: [ + ...MULTI_LANGUAGE_BASE, + column('language', 'STRING'), + column('qualifiedName', 'STRING'), + column('parentEnum', 'STRING'), + column('moduleQualifiedName', 'STRING'), + column('variantKind', 'STRING'), + column('fieldsJson', 'STRING'), + column('attributes', 'STRING[]', 'string-array'), + column('attributesJson', 'STRING'), + column('locationFidelity', 'STRING'), + ], +}; + +const CONST_LAYOUT: NodeTableLayout = { + table: 'Const', + quoteTableName: true, + columns: [ + ...MULTI_LANGUAGE_BASE, + column('language', 'STRING'), + column('qualifiedName', 'STRING'), + column('moduleQualifiedName', 'STRING'), + column('constType', 'STRING'), + column('constValue', 'STRING'), + column('isErrorCode', 'BOOLEAN', 'boolean', false), + column('locationFidelity', 'STRING'), + ], +}; + +const MODULE_LAYOUT: NodeTableLayout = { + table: 'Module', + quoteTableName: true, + columns: [ + ...MULTI_LANGUAGE_BASE, + column('language', 'STRING'), + column('qualifiedName', 'STRING'), + column('moduleAddress', 'STRING'), + column('attributes', 'STRING[]', 'string-array'), + column('attributesJson', 'STRING'), + column('locationFidelity', 'STRING'), + ], +}; + +export const NODE_TABLE_LAYOUTS = { + Function: FUNCTION_LAYOUT, + Struct: structLikeLayout('Struct'), + Enum: structLikeLayout('Enum'), + EnumVariant: ENUM_VARIANT_LAYOUT, + Const: CONST_LAYOUT, + Module: MODULE_LAYOUT, +} as const satisfies Partial>; + +export type LayoutTableName = keyof typeof NODE_TABLE_LAYOUTS; + +export const hasNodeTableLayout = (table: string): table is LayoutTableName => + Object.hasOwn(NODE_TABLE_LAYOUTS, table); + +export const getNodeTableLayout = (table: LayoutTableName): NodeTableLayout => + NODE_TABLE_LAYOUTS[table]; + +export const getNodeTableColumnNames = (table: NodeTableName): readonly string[] | undefined => + hasNodeTableLayout(table) ? getNodeTableLayout(table).columns.map(({ name }) => name) : undefined; + +export const getNodeTableCsvHeader = (table: LayoutTableName): string => + NODE_TABLE_LAYOUTS[table].columns.map(({ name }) => name).join(','); + +export const buildNodeTableSchema = (table: LayoutTableName): string => { + const layout = NODE_TABLE_LAYOUTS[table]; + const tableName = layout.quoteTableName ? `\`${layout.table}\`` : layout.table; + const columns = layout.columns.map(({ name, ddlType }) => ` ${name} ${ddlType},`).join('\n'); + return `\nCREATE NODE TABLE ${tableName} (\n${columns}\n PRIMARY KEY (id)\n)`; +}; diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index adbfd1bde..fe751ef20 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -11,6 +11,7 @@ // Import from shared package (single source of truth) — used in DDL templates below import { NODE_TABLES, REL_TABLE_NAME, REL_TYPES, EMBEDDING_TABLE_NAME } from 'gitnexus-shared'; +import { buildNodeTableSchema } from './node-table-layout.js'; // Re-export so downstream consumers keep the same import path export { NODE_TABLES, REL_TABLE_NAME, REL_TYPES, EMBEDDING_TABLE_NAME }; export type { NodeTableName, RelType } from 'gitnexus-shared'; @@ -36,35 +37,7 @@ CREATE NODE TABLE Folder ( PRIMARY KEY (id) )`; -export const FUNCTION_SCHEMA = ` -CREATE NODE TABLE Function ( - id STRING, - name STRING, - filePath STRING, - startLine INT64, - endLine INT64, - isExported BOOLEAN, - content STRING, - description STRING, - language STRING, - qualifiedName STRING, - moduleQualifiedName STRING, - visibility STRING, - visibilityModifier STRING, - isEntry BOOLEAN, - isView BOOLEAN, - isInline BOOLEAN, - isNative BOOLEAN, - parameterCount INT32, - returnType STRING, - acquires STRING[], - usedTypes STRING[], - attributes STRING[], - attributesJson STRING, - typeParamsJson STRING, - locationFidelity STRING, - PRIMARY KEY (id) -)`; +export const FUNCTION_SCHEMA = buildNodeTableSchema('Function'); export const CLASS_SCHEMA = ` CREATE NODE TABLE Class ( @@ -172,93 +145,9 @@ CREATE NODE TABLE \`${name}\` ( PRIMARY KEY (id) )`; -// Move struct/enum carry compiler-sourced abilities/resource/event/field facts. -const MOVE_STRUCT_LIKE_SCHEMA = (name: string) => ` -CREATE NODE TABLE \`${name}\` ( - id STRING, - name STRING, - filePath STRING, - startLine INT64, - endLine INT64, - content STRING, - description STRING, - language STRING, - qualifiedName STRING, - moduleQualifiedName STRING, - moduleAddress STRING, - abilities STRING[], - isResource BOOLEAN, - isEvent BOOLEAN, - fieldList STRING[], - attributes STRING[], - attributesJson STRING, - typeParamsJson STRING, - moveDeclarationKind STRING, - locationFidelity STRING, - PRIMARY KEY (id) -)`; - -const MOVE_ENUM_VARIANT_SCHEMA = ` -CREATE NODE TABLE \`EnumVariant\` ( - id STRING, - name STRING, - filePath STRING, - startLine INT64, - endLine INT64, - content STRING, - description STRING, - language STRING, - qualifiedName STRING, - parentEnum STRING, - moduleQualifiedName STRING, - variantKind STRING, - fieldsJson STRING, - attributes STRING[], - attributesJson STRING, - locationFidelity STRING, - PRIMARY KEY (id) -)`; - -const MOVE_MODULE_SCHEMA = ` -CREATE NODE TABLE \`Module\` ( - id STRING, - name STRING, - filePath STRING, - startLine INT64, - endLine INT64, - content STRING, - description STRING, - language STRING, - qualifiedName STRING, - moduleAddress STRING, - attributes STRING[], - attributesJson STRING, - locationFidelity STRING, - PRIMARY KEY (id) -)`; - -const MOVE_CONST_SCHEMA = ` -CREATE NODE TABLE \`Const\` ( - id STRING, - name STRING, - filePath STRING, - startLine INT64, - endLine INT64, - content STRING, - description STRING, - language STRING, - qualifiedName STRING, - moduleQualifiedName STRING, - constType STRING, - constValue STRING, - isErrorCode BOOLEAN, - locationFidelity STRING, - PRIMARY KEY (id) -)`; - -export const STRUCT_SCHEMA = MOVE_STRUCT_LIKE_SCHEMA('Struct'); -export const ENUM_SCHEMA = MOVE_STRUCT_LIKE_SCHEMA('Enum'); -export const ENUM_VARIANT_SCHEMA = MOVE_ENUM_VARIANT_SCHEMA; +export const STRUCT_SCHEMA = buildNodeTableSchema('Struct'); +export const ENUM_SCHEMA = buildNodeTableSchema('Enum'); +export const ENUM_VARIANT_SCHEMA = buildNodeTableSchema('EnumVariant'); export const MACRO_SCHEMA = CODE_ELEMENT_BASE('Macro'); export const TYPEDEF_SCHEMA = CODE_ELEMENT_BASE('Typedef'); export const UNION_SCHEMA = CODE_ELEMENT_BASE('Union'); @@ -266,7 +155,7 @@ export const NAMESPACE_SCHEMA = CODE_ELEMENT_BASE('Namespace'); export const TRAIT_SCHEMA = CODE_ELEMENT_BASE('Trait'); export const IMPL_SCHEMA = CODE_ELEMENT_BASE('Impl'); export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias'); -export const CONST_SCHEMA = MOVE_CONST_SCHEMA; +export const CONST_SCHEMA = buildNodeTableSchema('Const'); export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static'); export const VARIABLE_SCHEMA = CODE_ELEMENT_BASE('Variable'); export const PROPERTY_SCHEMA = ` @@ -286,7 +175,7 @@ export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate'); export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation'); export const CONSTRUCTOR_SCHEMA = CODE_ELEMENT_BASE('Constructor'); export const TEMPLATE_SCHEMA = CODE_ELEMENT_BASE('Template'); -export const MODULE_SCHEMA = MOVE_MODULE_SCHEMA; +export const MODULE_SCHEMA = buildNodeTableSchema('Module'); // API route endpoints (Next.js, Express, etc.) export const ROUTE_SCHEMA = ` CREATE NODE TABLE Route ( diff --git a/gitnexus/src/core/move/install.ts b/gitnexus/src/core/move/install.ts new file mode 100644 index 000000000..4d724b932 --- /dev/null +++ b/gitnexus/src/core/move/install.ts @@ -0,0 +1,663 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { createReadStream, createWriteStream, type Dirent, type Stats } from 'node:fs'; +import { + chmod, + copyFile, + type FileHandle, + mkdir, + mkdtemp, + open, + readFile, + readdir, + rename, + rm, + stat, + utimes, + writeFile, +} from 'node:fs/promises'; +import type { ClientRequest, IncomingMessage } from 'node:http'; +import { get as httpsGet, type RequestOptions } from 'node:https'; +import os from 'node:os'; +import path from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { MOVE_FLOW_RELEASE } from './release.js'; + +export type MoveFlowInstallStatus = + | 'available' + | 'installed' + | 'skipped' + | 'unsupported' + | 'failed'; + +export interface MoveFlowInstallResult { + status: MoveFlowInstallStatus; + binaryPath?: string; + message?: string; +} + +export interface MoveFlowInstallConfig { + version: string; + repository: string; + tag: string; + releaseTarget: string; + assetName: string; + releaseBase: string; + binaryName: string; + installDir: string; + binaryPath: string; + metadataPath: string; + lockPath: string; + httpTimeoutMs: number; +} + +interface MoveFlowCacheMetadata { + version: string; + repository: string; + tag: string; + assetName: string; + archiveSha256: string; + binarySha256: string; +} + +export interface LockOptions { + waitTimeoutMs?: number; + leaseMs?: number; + heartbeatMs?: number; + retryMs?: number; +} + +export interface InstallLockIo { + openLock(lockPath: string): Promise; + removeLock(lockPath: string): Promise; +} + +interface PowerShellInvocation { + args: string[]; + env: NodeJS.ProcessEnv; +} + +type HttpsGet = ( + url: string | URL, + options: RequestOptions, + callback: (response: IncomingMessage) => void, +) => ClientRequest; + +const DEFAULT_HTTP_TIMEOUT_MS = 30_000; +const DEFAULT_LOCK_WAIT_MS = 60_000; +const INSTALL_COMPLETION_GRACE_MS = 60_000; +const DEFAULT_LOCK_LEASE_MS = 60_000; +const DEFAULT_HEARTBEAT_MS = 5_000; +const DEFAULT_LOCK_RETRY_MS = 100; + +const defaultLockIo: InstallLockIo = { + openLock: (lockPath) => open(lockPath, 'wx'), + removeLock: async (lockPath) => { + await rm(lockPath, { force: true }); + }, +}; + +const wait = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +const parsePositiveInteger = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; +}; + +/** + * A waiter must outlive a healthy cold install: two sequential downloads plus + * checksum, extraction, copy, and the executable version probe. + */ +export const moveFlowInstallLockWaitMs = (httpTimeoutMs: number): number => + Math.max(DEFAULT_LOCK_WAIT_MS, httpTimeoutMs * 2 + INSTALL_COMPLETION_GRACE_MS); + +const validateReleaseCoordinates = (version: string, repository: string, tag: string): void => { + if (!/^\d+\.\d+\.\d+$/.test(version)) { + throw new Error(`invalid move-flow version '${version}'; expected X.Y.Z`); + } + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) { + throw new Error(`invalid move-flow repository '${repository}'; expected owner/name`); + } + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(tag)) { + throw new Error(`invalid move-flow release tag '${tag}'`); + } +}; + +const platformKey = (platform = process.platform, arch = process.arch): string | null => { + if (platform === 'linux' && arch === 'x64') return 'linux-x64'; + if (platform === 'linux' && arch === 'arm64') return 'linux-arm64'; + if (platform === 'darwin' && arch === 'arm64') return 'darwin-arm64'; + if (platform === 'darwin' && arch === 'x64') return 'darwin-x64'; + if (platform === 'win32' && arch === 'x64') return 'win32-x64'; + return null; +}; + +const execFileOutput = ( + file: string, + args: string[], + options: { timeout: number; env?: NodeJS.ProcessEnv; encoding?: BufferEncoding }, +): Promise => + new Promise((resolve, reject) => { + execFile( + file, + args, + { + timeout: options.timeout, + env: options.env, + encoding: options.encoding ?? 'utf8', + }, + (error, stdout) => { + if (error) reject(error); + else resolve(String(stdout)); + }, + ); + }); + +const linuxNeedsCompatBuild = async (env: NodeJS.ProcessEnv): Promise => { + if (process.platform !== 'linux') return false; + if (env.GITNEXUS_MOVE_FLOW_COMPAT === '1') return true; + + if (process.arch === 'x64') { + try { + const cpuinfo = await readFile('/proc/cpuinfo', 'utf8'); + if (!/(^|\s)avx2(\s|$)/m.test(cpuinfo)) return true; + } catch { + return true; + } + } + + try { + const output = await execFileOutput('ldd', ['--version'], { timeout: 3_000 }); + const match = /(\d+)\.(\d+)/.exec(output.split('\n')[0] ?? ''); + if (!match) return false; + const major = Number(match[1]); + const minor = Number(match[2]); + return major < 2 || (major === 2 && minor < 34); + } catch { + return false; + } +}; + +const releaseTargetForPlatform = async ( + key: string, + env: NodeJS.ProcessEnv, +): Promise => { + switch (key) { + case 'darwin-arm64': + return 'aarch64-apple-darwin'; + case 'darwin-x64': + return 'x86_64-apple-darwin'; + case 'linux-arm64': + return `aarch64-unknown-linux-gnu${(await linuxNeedsCompatBuild(env)) ? '-compat' : ''}`; + case 'linux-x64': + return `x86_64-unknown-linux-gnu${(await linuxNeedsCompatBuild(env)) ? '-compat' : ''}`; + case 'win32-x64': + return 'x86_64-pc-windows-msvc'; + default: + return null; + } +}; + +export async function getMoveFlowInstallConfig( + env: NodeJS.ProcessEnv = process.env, +): Promise { + const key = platformKey(); + if (!key) return null; + + const version = env.GITNEXUS_MOVE_FLOW_VERSION?.trim() || MOVE_FLOW_RELEASE.version; + const repository = env.GITNEXUS_MOVE_FLOW_REPO?.trim() || MOVE_FLOW_RELEASE.repository; + const tag = env.GITNEXUS_MOVE_FLOW_TAG?.trim() || `${MOVE_FLOW_RELEASE.tagPrefix}${version}`; + validateReleaseCoordinates(version, repository, tag); + const releaseTarget = await releaseTargetForPlatform(key, env); + if (!releaseTarget) return null; + + const assetName = `${tag}-${releaseTarget}.zip`; + if (path.basename(assetName) !== assetName) throw new Error('invalid move-flow asset name'); + const identity = createHash('sha256') + .update(`${repository}\0${tag}\0${assetName}`) + .digest('hex') + .slice(0, 12); + const cacheRoot = env.GITNEXUS_MOVE_FLOW_DIR?.trim() + ? path.resolve(env.GITNEXUS_MOVE_FLOW_DIR) + : path.join( + env.GITNEXUS_HOME?.trim() || path.join(os.homedir(), '.gitnexus'), + 'tools', + 'move-flow', + ); + const installDir = path.join(cacheRoot, version, `${key}-${identity}`); + const binaryName = process.platform === 'win32' ? 'move-flow.exe' : 'move-flow'; + + return { + version, + repository, + tag, + releaseTarget, + assetName, + releaseBase: `https://github.com/${repository}/releases/download/${tag}`, + binaryName, + installDir, + binaryPath: path.join(installDir, binaryName), + metadataPath: path.join(installDir, 'release.json'), + lockPath: `${installDir}.install.lock`, + httpTimeoutMs: parsePositiveInteger( + env.GITNEXUS_MOVE_FLOW_HTTP_TIMEOUT_MS, + DEFAULT_HTTP_TIMEOUT_MS, + ), + }; +} + +export const sha256File = async (file: string): Promise => { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(file)) hash.update(chunk); + return hash.digest('hex'); +}; + +export const expectedSha = (sumsText: string, assetName: string): string | null => { + for (const raw of sumsText.split('\n')) { + const line = raw.trim(); + if (!line) continue; + const standard = /^([0-9a-f]{64})[ \t*]+(\S.*)$/i.exec(line); + if (standard && standard[2] === assetName) return standard[1].toLowerCase(); + const bsd = /^SHA256\s*\((.*)\)\s*=\s*([0-9a-f]{64})$/i.exec(line); + if (bsd && bsd[1] === assetName) return bsd[2].toLowerCase(); + } + return null; +}; + +export type ChecksumVerification = + | { status: 'match'; expected: string; actual: string } + | { status: 'mismatch'; expected: string; actual: string } + | { status: 'missing' }; + +export const verifyArchiveChecksum = async ( + sumsText: string, + assetName: string, + archivePath: string, +): Promise => { + const expected = expectedSha(sumsText, assetName); + if (!expected) return { status: 'missing' }; + const actual = await sha256File(archivePath); + return actual === expected + ? { status: 'match', expected, actual } + : { status: 'mismatch', expected, actual }; +}; + +export const powershellExpandArchiveInvocation = ( + archive: string, + destination: string, +): PowerShellInvocation => { + const archiveEnv = 'GITNEXUS_MOVE_FLOW_ARCHIVE_PATH'; + const destinationEnv = 'GITNEXUS_MOVE_FLOW_DESTINATION_PATH'; + return { + args: [ + '-NoProfile', + '-NonInteractive', + '-Command', + `Expand-Archive -LiteralPath $env:${archiveEnv} -DestinationPath $env:${destinationEnv} -Force`, + ], + env: { + ...process.env, + [archiveEnv]: archive, + [destinationEnv]: destination, + }, + }; +}; + +export const downloadToFile = async ( + url: string, + destination: string, + timeoutMs: number, + get: HttpsGet = httpsGet, +): Promise => { + const partial = `${destination}.partial`; + const deadline = Date.now() + timeoutMs; + + const request = (target: string, redirects: number): Promise => + new Promise((resolve, reject) => { + if (redirects > 5) { + reject(new Error('too many redirects')); + return; + } + + const req = get(target, {}, (response) => { + const status = response.statusCode ?? 0; + if (status >= 300 && status < 400 && response.headers.location) { + response.once('error', reject); + response.resume(); + request(new URL(response.headers.location, target).toString(), redirects + 1).then( + resolve, + reject, + ); + return; + } + if (status !== 200) { + response.once('error', reject); + response.resume(); + reject(new Error(`HTTP ${status} for ${target}`)); + return; + } + + pipeline(response, createWriteStream(partial)) + .then(() => rename(partial, destination)) + .then(resolve, reject); + }); + const timeout = setTimeout( + () => req.destroy(new Error('download timed out')), + Math.max(1, deadline - Date.now()), + ); + req.once('close', () => clearTimeout(timeout)); + req.on('error', reject); + }); + + try { + await request(url, 0); + } catch (error) { + await rm(partial, { force: true }); + throw error; + } +}; + +const readLock = async (lockPath: string): Promise => { + try { + const parsed = JSON.parse(await readFile(lockPath, 'utf8')) as { token?: unknown }; + return typeof parsed.token === 'string' ? parsed.token : null; + } catch { + return null; + } +}; + +const statOrNull = async (file: string): Promise => { + try { + return await stat(file); + } catch { + return null; + } +}; + +const removeStaleLock = async (lockPath: string, leaseMs: number): Promise => { + const first = await statOrNull(lockPath); + if (!first || Date.now() - first.mtimeMs <= leaseMs) return; + const token = await readLock(lockPath); + + await wait(25); + const second = await statOrNull(lockPath); + if ( + !second || + second.mtimeMs !== first.mtimeMs || + Date.now() - second.mtimeMs <= leaseMs || + (await readLock(lockPath)) !== token + ) { + return; + } + await rm(lockPath, { force: true }); +}; + +export async function acquireInstallLock( + lockPath: string, + options: LockOptions = {}, + io: InstallLockIo = defaultLockIo, +): Promise<() => Promise> { + const waitTimeoutMs = options.waitTimeoutMs ?? DEFAULT_LOCK_WAIT_MS; + const leaseMs = options.leaseMs ?? DEFAULT_LOCK_LEASE_MS; + const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS; + const retryMs = options.retryMs ?? DEFAULT_LOCK_RETRY_MS; + const deadline = Date.now() + waitTimeoutMs; + const token = randomUUID(); + + await mkdir(path.dirname(lockPath), { recursive: true }); + while (Date.now() < deadline) { + let created = false; + try { + const handle = await io.openLock(lockPath); + created = true; + try { + await handle.writeFile(JSON.stringify({ token, pid: process.pid })); + } finally { + await handle.close(); + } + + let heartbeatRunning = false; + const heartbeat = setInterval(() => { + if (heartbeatRunning) return; + heartbeatRunning = true; + void (async () => { + if ((await readLock(lockPath)) !== token) return; + const now = new Date(); + await utimes(lockPath, now, now); + })() + .catch(() => {}) + .finally(() => { + heartbeatRunning = false; + }); + }, heartbeatMs); + heartbeat.unref(); + + return async () => { + clearInterval(heartbeat); + if ((await readLock(lockPath)) === token) await rm(lockPath, { force: true }); + }; + } catch (error) { + if (created) { + await io.removeLock(lockPath).catch(() => {}); + } + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + await removeStaleLock(lockPath, leaseMs); + await wait(retryMs); + } + } + + throw new Error('timed out waiting for another move-flow installation'); +} + +export async function settleMoveFlowCleanup( + ...tasks: Array<(() => Promise) | undefined> +): Promise { + const active = tasks.filter((task): task is () => Promise => task !== undefined); + await Promise.allSettled(active.map((task) => Promise.resolve().then(task))); +} + +export const reportedMoveFlowVersion = (output: string): string | null => + /(?:^|\s)v?(\d+\.\d+\.\d+)(?:\s|$)/.exec(output)?.[1] ?? null; + +const exactVersionMatches = async (binaryPath: string, version: string): Promise => { + try { + const output = await execFileOutput(binaryPath, ['--version'], { timeout: 5_000 }); + return reportedMoveFlowVersion(output) === version; + } catch { + return false; + } +}; + +const expectedMetadata = ( + config: MoveFlowInstallConfig, +): Pick => ({ + version: config.version, + repository: config.repository, + tag: config.tag, + assetName: config.assetName, +}); + +const validCachedInstall = async (config: MoveFlowInstallConfig): Promise => { + try { + const metadata = JSON.parse( + await readFile(config.metadataPath, 'utf8'), + ) as MoveFlowCacheMetadata; + const expected = expectedMetadata(config); + if ( + metadata.version !== expected.version || + metadata.repository !== expected.repository || + metadata.tag !== expected.tag || + metadata.assetName !== expected.assetName || + metadata.binarySha256 !== (await sha256File(config.binaryPath)) + ) { + return false; + } + return exactVersionMatches(config.binaryPath, config.version); + } catch { + return false; + } +}; + +export async function findCachedMoveFlow( + env: NodeJS.ProcessEnv = process.env, +): Promise { + try { + const config = await getMoveFlowInstallConfig(env); + return config && (await validCachedInstall(config)) ? config.binaryPath : null; + } catch { + return null; + } +} + +const extractZip = async (archive: string, destination: string): Promise => { + await mkdir(destination, { recursive: true }); + try { + if (process.platform === 'win32') { + const shell = process.env.ComSpec ? 'powershell.exe' : 'powershell'; + const invocation = powershellExpandArchiveInvocation(archive, destination); + await execFileOutput(shell, invocation.args, { timeout: 30_000, env: invocation.env }); + return; + } + await execFileOutput('unzip', ['-q', archive, '-d', destination], { timeout: 30_000 }); + } catch (error) { + throw new Error( + `could not extract ${path.basename(archive)} (${error instanceof Error ? error.message : String(error)}). ` + + 'Install unzip, or set MOVE_FLOW to an existing move-flow binary.', + ); + } +}; + +const findExtractedBinary = async (root: string, binaryName: string): Promise => { + const stack = [root]; + const names = new Set([binaryName, 'move-flow']); + while (stack.length > 0) { + const current = stack.pop(); + if (!current) break; + const entries: Dirent[] = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (names.has(entry.name)) return full; + } + } + return null; +}; + +export async function installMoveFlow( + env: NodeJS.ProcessEnv = process.env, +): Promise { + // GITNEXUS_SKIP_OPTIONAL_GRAMMARS is the umbrella opt-out for all optional + // binary downloads (honored by the retired postinstall probe too). + for (const flag of ['GITNEXUS_SKIP_MOVE_FLOW', 'GITNEXUS_SKIP_OPTIONAL_GRAMMARS'] as const) { + if (env[flag] === '1') { + return { status: 'skipped', message: `installation disabled by ${flag}=1` }; + } + } + + let config: MoveFlowInstallConfig | null; + try { + config = await getMoveFlowInstallConfig(env); + } catch (error) { + return { + status: 'failed', + message: error instanceof Error ? error.message : String(error), + }; + } + if (!config) { + return { + status: 'unsupported', + message: `unsupported platform ${process.platform}-${process.arch}; set MOVE_FLOW to provide a binary`, + }; + } + if (await validCachedInstall(config)) { + return { status: 'available', binaryPath: config.binaryPath }; + } + + let releaseLock: (() => Promise) | undefined; + let tempDir: string | undefined; + let stagedDir: string | undefined; + try { + releaseLock = await acquireInstallLock(config.lockPath, { + waitTimeoutMs: moveFlowInstallLockWaitMs(config.httpTimeoutMs), + }); + if (await validCachedInstall(config)) { + return { status: 'available', binaryPath: config.binaryPath }; + } + + tempDir = await mkdtemp(path.join(os.tmpdir(), 'move-flow-')); + const archivePath = path.join(tempDir, config.assetName); + const sumsPath = path.join(tempDir, 'SHA256SUMS'); + const extractDir = path.join(tempDir, 'extract'); + await downloadToFile(`${config.releaseBase}/SHA256SUMS`, sumsPath, config.httpTimeoutMs); + await downloadToFile( + `${config.releaseBase}/${config.assetName}`, + archivePath, + config.httpTimeoutMs, + ); + + const verification = await verifyArchiveChecksum( + await readFile(sumsPath, 'utf8'), + config.assetName, + archivePath, + ); + if (verification.status === 'missing') { + return { + status: 'failed', + message: `SHA256SUMS does not list ${config.assetName}; refusing to install`, + }; + } + if (verification.status === 'mismatch') { + return { + status: 'failed', + message: `checksum mismatch for ${config.assetName}; refusing to install`, + }; + } + + await extractZip(archivePath, extractDir); + const extracted = await findExtractedBinary(extractDir, config.binaryName); + if (!extracted) { + return { + status: 'failed', + message: `${config.assetName} did not contain ${config.binaryName}; refusing to install`, + }; + } + + await mkdir(path.dirname(config.installDir), { recursive: true }); + stagedDir = await mkdtemp(`${config.installDir}.partial-`); + const stagedBinary = path.join(stagedDir, config.binaryName); + await copyFile(extracted, stagedBinary); + if (process.platform !== 'win32') await chmod(stagedBinary, 0o755); + if (!(await exactVersionMatches(stagedBinary, config.version))) { + return { + status: 'failed', + message: `installed binary did not report exact version ${config.version}; refusing to keep it`, + }; + } + + const metadata: MoveFlowCacheMetadata = { + ...expectedMetadata(config), + archiveSha256: verification.actual, + binarySha256: await sha256File(stagedBinary), + }; + await writeFile(path.join(stagedDir, 'release.json'), JSON.stringify(metadata, null, 2)); + + await rm(config.installDir, { recursive: true, force: true }); + await rename(stagedDir, config.installDir); + stagedDir = undefined; + return { status: 'installed', binaryPath: config.binaryPath }; + } catch (error) { + return { + status: 'failed', + message: `installation failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } finally { + await settleMoveFlowCleanup( + tempDir ? () => rm(tempDir, { recursive: true, force: true }) : undefined, + stagedDir ? () => rm(stagedDir, { recursive: true, force: true }) : undefined, + releaseLock, + ); + } +} diff --git a/gitnexus/src/core/move/mcp-client.ts b/gitnexus/src/core/move/mcp-client.ts index 7c05ba9b4..828621321 100644 --- a/gitnexus/src/core/move/mcp-client.ts +++ b/gitnexus/src/core/move/mcp-client.ts @@ -8,10 +8,8 @@ import { spawn, execFileSync, type ChildProcessWithoutNullStreams } from 'node:child_process'; import { createInterface } from 'node:readline'; -import { existsSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import * as path from 'node:path'; import type { MoveFactsMap, CallGraphMap } from './compiler-facts.js'; +import { MOVE_FLOW_RELEASE } from './release.js'; interface JsonRpcResponse { jsonrpc: '2.0'; @@ -550,49 +548,41 @@ export class MoveFlowMcpClient implements MoveFlowClient { /** * Try to create a MoveFlowMcpClient. Returns null if move-flow binary - * is not found on the system. + * is not found on the system. When `binaryPath` is provided, only that + * provisioned path is probed; the normal resolution order is bypassed. * * Resolution order: * 1. `$MOVE_FLOW` (explicit override for power users / CI). - * 2. Bundled `vendor/move-flow//move-flow[.exe]` (the postinstall - * probe installs here — see `scripts/install-move-flow.cjs`). - * 3. `move-flow` on `$PATH` (host install). + * 2. `move-flow` on `$PATH` (host install). */ -function bundledMoveFlowPath(): string | null { - const { platform, arch } = process; - let key: string | null = null; - if (platform === 'linux' && arch === 'x64') key = 'linux-x64'; - else if (platform === 'linux' && arch === 'arm64') key = 'linux-arm64'; - else if (platform === 'darwin' && arch === 'arm64') key = 'darwin-arm64'; - else if (platform === 'darwin' && arch === 'x64') key = 'darwin-x64'; - else if (platform === 'win32' && arch === 'x64') key = 'win32-x64'; - if (!key) return null; - const name = platform === 'win32' ? 'move-flow.exe' : 'move-flow'; - // mcp-client.ts lives at gitnexus/src/core/move/; vendor/ is two levels above src/. - const here = path.dirname(fileURLToPath(import.meta.url)); - return path.resolve(here, '..', '..', '..', 'vendor', 'move-flow', key, name); -} +const MOVE_FLOW_COMPATIBLE_MAJOR = Number(MOVE_FLOW_RELEASE.version.split('.')[0]); function probeBinary(binary: string): boolean { try { - execFileSync(binary, ['--version'], { stdio: 'ignore', timeout: 5000 }); - return true; + const output = execFileSync(binary, ['--version'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 5000, + }); + // Prerelease/build suffixes ("2.1.0-rc1") are accepted; only the major + // version gates protocol compatibility, and it follows the release pin. + const version = /(?:^|\s)v?(\d+)\.(\d+)\.(\d+)(?:[-+][0-9A-Za-z.-]+)?(?:\s|$)/.exec(output); + return version !== null && Number(version[1]) === MOVE_FLOW_COMPATIBLE_MAJOR; } catch { return false; } } -export function tryCreateMoveFlowClient(): MoveFlowMcpClient | null { +export function tryCreateMoveFlowClient(binaryPath?: string): MoveFlowMcpClient | null { + if (binaryPath) { + return probeBinary(binaryPath) ? new MoveFlowMcpClient(binaryPath) : null; + } + const explicit = process.env.MOVE_FLOW; if (explicit) { return probeBinary(explicit) ? new MoveFlowMcpClient(explicit) : null; } - const bundled = bundledMoveFlowPath(); - if (bundled && existsSync(bundled) && probeBinary(bundled)) { - return new MoveFlowMcpClient(bundled); - } - const onPath = 'move-flow'; return probeBinary(onPath) ? new MoveFlowMcpClient(onPath) : null; } diff --git a/gitnexus/src/core/move/provision.ts b/gitnexus/src/core/move/provision.ts new file mode 100644 index 000000000..a80fe7c64 --- /dev/null +++ b/gitnexus/src/core/move/provision.ts @@ -0,0 +1,88 @@ +import { MoveFlowMcpClient, tryCreateMoveFlowClient } from './mcp-client.js'; +import { findCachedMoveFlow, installMoveFlow, type MoveFlowInstallResult } from './install.js'; +import { MOVE_FLOW_RELEASE } from './release.js'; + +// One install attempt per dependency set per process: concurrent callers share +// the in-flight promise, and a settled failure is not retried until restart +// (an explicit MOVE_FLOW or a PATH/cache binary is still re-probed every call). +const installAttempts = new WeakMap< + MoveFlowProvisionDependencies, + Promise +>(); + +export interface MoveFlowProvisionOptions { + onLog?: (message: string) => void; +} + +export interface MoveFlowProvisionDependencies { + resolveClient: (binaryPath?: string) => MoveFlowMcpClient | null; + findCached: () => Promise; + install: () => Promise; +} + +const defaultDependencies: MoveFlowProvisionDependencies = { + resolveClient: tryCreateMoveFlowClient, + findCached: findCachedMoveFlow, + install: installMoveFlow, +}; + +/** + * Resolve move-flow, provisioning the pinned release only when a caller has + * already detected Move code. Explicit MOVE_FLOW overrides remain authoritative: + * an invalid override is reported instead of silently replaced. + */ +export async function ensureMoveFlowClient( + options: MoveFlowProvisionOptions = {}, + dependencies: MoveFlowProvisionDependencies = defaultDependencies, +): Promise { + if (process.env.MOVE_FLOW) { + const explicit = dependencies.resolveClient(process.env.MOVE_FLOW); + if (explicit) return explicit; + options.onLog?.( + `MOVE_FLOW points to an unavailable binary (${process.env.MOVE_FLOW}); automatic installation was skipped.`, + ); + return null; + } + + const cachedPath = await dependencies.findCached(); + if (cachedPath) { + const cached = dependencies.resolveClient(cachedPath); + if (cached) return cached; + } + + const existing = dependencies.resolveClient(); + if (existing) return existing; + + let attempt = installAttempts.get(dependencies); + if (!attempt) { + options.onLog?.( + `Move code detected; ensuring move-flow ${MOVE_FLOW_RELEASE.version} is available.`, + ); + attempt = dependencies.install(); + installAttempts.set(dependencies, attempt); + } + + let result: MoveFlowInstallResult; + try { + result = await attempt; + } catch (err) { + options.onLog?.( + `move-flow installation failed: ${err instanceof Error ? err.message : String(err)}; ` + + '.move files will not be indexed (not retried in this process).', + ); + return null; + } + + if (!result.binaryPath) { + options.onLog?.( + `move-flow is unavailable${result.message ? `: ${result.message}` : ''}; .move files will not be indexed.`, + ); + return null; + } + + const installed = dependencies.resolveClient(result.binaryPath); + if (installed) return installed; + + options.onLog?.(`Installed move-flow failed its runtime probe at ${result.binaryPath}.`); + return null; +} diff --git a/gitnexus/src/core/move/release.ts b/gitnexus/src/core/move/release.ts new file mode 100644 index 000000000..1aa9dd419 --- /dev/null +++ b/gitnexus/src/core/move/release.ts @@ -0,0 +1,5 @@ +export const MOVE_FLOW_RELEASE = { + version: '2.0.0', + repository: 'aptos-labs/aptos-ai', + tagPrefix: 'move-flow-v', +} as const; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index d956bd991..c1c421594 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -15,8 +15,8 @@ import { execFileSync } from 'child_process'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; import { isMoveCompilerInputPath, moveAvailabilityRequiresFullRebuild } from './move/constants.js'; import { createMoveIngestPhase } from './move/move-ingest.js'; -import { tryCreateMoveFlowClient } from './move/mcp-client.js'; import { repoHasMove } from './move/discovery.js'; +import { ensureMoveFlowClient } from './move/provision.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { initLbug, @@ -944,7 +944,7 @@ export async function runFullAnalysis( // Probe before the same-commit fast path: an index built while move-flow was // unavailable must be backfilled once the compiler becomes available. const hasMovePackages = await repoHasMove(repoPath); - const moveFlowClient = hasMovePackages ? tryCreateMoveFlowClient() : null; + const moveFlowClient = hasMovePackages ? await ensureMoveFlowClient({ onLog: log }) : null; const moveIngestAvailable = hasMovePackages ? moveFlowClient !== null : undefined; if ( existingMeta && diff --git a/gitnexus/test/integration/move-live.test.ts b/gitnexus/test/integration/move-live.test.ts index 4b4a8ca9a..78b36b276 100644 --- a/gitnexus/test/integration/move-live.test.ts +++ b/gitnexus/test/integration/move-live.test.ts @@ -1,10 +1,12 @@ /** * Live end-to-end Move ingestion against the real move-flow binary. * - * Gated on move-flow being installed (skipped in CI without the binary). Proves - * the full compiler-first chain: capability probe → facts query → thin - * facts→graph mapper → pipeline graph, with full fidelity (resource/friend - * edges, resource structs, precise locations). + * Locally this suite skips when move-flow is unavailable. CI sets + * GITNEXUS_REQUIRE_MOVE_FLOW=1, which exercises on-demand provisioning and + * makes an unavailable release a hard failure. Proves the full compiler-first + * chain: capability probe → facts query → thin facts→graph mapper → + * pipeline graph, with full fidelity (resource/friend edges, resource structs, + * precise locations). */ import { describe, it, expect, afterAll } from 'vitest'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; @@ -13,9 +15,16 @@ import path from 'node:path'; import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; import { tryCreateMoveFlowClient } from '../../src/core/move/mcp-client.js'; import { createMoveIngestPhase } from '../../src/core/move/move-ingest.js'; +import { ensureMoveFlowClient } from '../../src/core/move/provision.js'; import { runMoveIngestPhase } from '../helpers/move-ingest-harness.js'; -const client = tryCreateMoveFlowClient(); +const requireMoveFlow = process.env.GITNEXUS_REQUIRE_MOVE_FLOW === '1'; +const client = requireMoveFlow + ? await ensureMoveFlowClient({ onLog: (message) => console.info(`[move-live] ${message}`) }) + : tryCreateMoveFlowClient(); +if (requireMoveFlow && !client) { + throw new Error('GITNEXUS_REQUIRE_MOVE_FLOW=1 but MoveFlow provisioning failed'); +} const coinFixture = path.resolve(process.cwd(), 'test/fixtures/move/aptos-framework/coin'); function writePackage(root: string, name: string, addr: string, source: string): void { diff --git a/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts b/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts new file mode 100644 index 000000000..4738634a8 --- /dev/null +++ b/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts @@ -0,0 +1,115 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; +import { buildTestGraph } from '../helpers/test-graph.js'; + +withTestLbugDB( + 'move-mixed-language-roundtrip', + () => { + describe('mixed-language persistence', () => { + it('stores Move facts and safe defaults for non-Move rows in shared tables', async () => { + const { executeQuery } = await import('../../src/core/lbug/lbug-adapter.js'); + + const functions = (await executeQuery( + 'MATCH (f:Function) RETURN f.id AS id, f.language AS language, ' + + 'f.qualifiedName AS qualifiedName, f.isEntry AS isEntry ORDER BY f.id', + )) as Array>; + expect(functions).toEqual([ + { + id: 'Function:move', + language: 'move', + qualifiedName: '0x1::coin::mint', + isEntry: true, + }, + { + id: 'Function:typescript', + language: 'typescript', + qualifiedName: null, + isEntry: false, + }, + ]); + + const structs = (await executeQuery( + 'MATCH (s:`Struct`) RETURN s.id AS id, s.language AS language, ' + + 's.qualifiedName AS qualifiedName, s.isResource AS isResource ORDER BY s.id', + )) as Array>; + expect(structs).toEqual([ + { + id: 'Struct:move', + language: 'move', + qualifiedName: '0x1::coin::CoinStore', + isResource: true, + }, + { + id: 'Struct:rust', + language: 'rust', + qualifiedName: null, + isResource: false, + }, + ]); + }); + }); + }, + { + beforeFTS: async (dbPath) => { + const repoPath = path.join(path.dirname(dbPath), 'repo'); + const storagePath = path.join(path.dirname(dbPath), 'storage'); + await fs.mkdir(repoPath, { recursive: true }); + + const graph = buildTestGraph([ + { + id: 'Function:move', + label: 'Function', + name: 'mint', + filePath: 'sources/coin.move', + startLine: 1, + endLine: 4, + isExported: true, + extra: { + language: 'move', + qualifiedName: '0x1::coin::mint', + moduleQualifiedName: '0x1::coin', + isEntry: true, + }, + }, + { + id: 'Function:typescript', + label: 'Function', + name: 'mint', + filePath: 'src/coin.ts', + startLine: 1, + endLine: 2, + isExported: true, + extra: { language: 'typescript' }, + }, + { + id: 'Struct:move', + label: 'Struct', + name: 'CoinStore', + filePath: 'sources/coin.move', + startLine: 6, + endLine: 10, + extra: { + language: 'move', + qualifiedName: '0x1::coin::CoinStore', + moduleQualifiedName: '0x1::coin', + isResource: true, + }, + }, + { + id: 'Struct:rust', + label: 'Struct', + name: 'CoinStore', + filePath: 'src/coin.rs', + startLine: 1, + endLine: 3, + extra: { language: 'rust' }, + }, + ]); + + const { loadGraphToLbug } = await import('../../src/core/lbug/lbug-adapter.js'); + await loadGraphToLbug(graph, repoPath, storagePath); + }, + }, +); diff --git a/gitnexus/test/unit/move/install-move-flow.test.ts b/gitnexus/test/unit/move/install-move-flow.test.ts index a560e63c8..d0fa11983 100644 --- a/gitnexus/test/unit/move/install-move-flow.test.ts +++ b/gitnexus/test/unit/move/install-move-flow.test.ts @@ -1,55 +1,32 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createRequire } from 'node:module'; import { EventEmitter } from 'node:events'; -import { PassThrough } from 'node:stream'; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import type { FileHandle } from 'node:fs/promises'; +import type { ClientRequest, IncomingMessage } from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; - -interface DownloadResponse extends PassThrough { - statusCode?: number; - headers: { location?: string }; -} - -type DownloadGet = (url: string, onResponse: (response: DownloadResponse) => void) => EventEmitter; - -type ChecksumVerification = - | { status: 'match'; expected: string; actual: string } - | { status: 'mismatch'; expected: string; actual: string } - | { status: 'missing' }; - -interface InstallerHelpers { - downloadToFile(url: string, dest: string, get?: DownloadGet): Promise; - expectedSha(sumsText: string, assetName: string): string | null; - sha256(file: string): string; - verifyArchiveChecksum(sumsText: string, assetName: string, archive: string): ChecksumVerification; - powershellExpandArchiveInvocation( - archive: string, - dest: string, - ): { - args: string[]; - env: NodeJS.ProcessEnv; - }; -} - -const require = createRequire(import.meta.url); -const { +import { PassThrough } from 'node:stream'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + acquireInstallLock, downloadToFile, expectedSha, - sha256, - verifyArchiveChecksum, + getMoveFlowInstallConfig, + installMoveFlow, + moveFlowInstallLockWaitMs, powershellExpandArchiveInvocation, -} = require('../../../scripts/install-move-flow.cjs') as InstallerHelpers; + reportedMoveFlowVersion, + settleMoveFlowCleanup, + sha256File, + verifyArchiveChecksum, +} from '../../../src/core/move/install.js'; const tempRoots: string[] = []; afterEach(() => { - for (const root of tempRoots.splice(0)) { - rmSync(root, { recursive: true, force: true }); - } + for (const root of tempRoots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -describe('install-move-flow', () => { +describe('move-flow installer', () => { const assetName = 'move-flow-v2.0.0-x86_64-unknown-linux-gnu.zip'; function writeArchive(contents = 'verified move-flow archive'): string { @@ -60,51 +37,43 @@ describe('install-move-flow', () => { return archive; } - it('accepts an exact asset entry whose checksum matches the downloaded archive', () => { + it('accepts only an exact asset entry whose checksum matches', async () => { const archive = writeArchive(); - const digest = sha256(archive); + const digest = await sha256File(archive); const sums = `${digest.toUpperCase()} ${assetName}\n`; expect(expectedSha(sums, assetName)).toBe(digest); - expect(verifyArchiveChecksum(sums, assetName, archive)).toEqual({ + await expect(verifyArchiveChecksum(sums, assetName, archive)).resolves.toEqual({ status: 'match', expected: digest, actual: digest, }); - // Preserve the release parser's supported BSD checksum format too. expect(expectedSha(`SHA256 (${assetName}) = ${digest.toUpperCase()}`, assetName)).toBe(digest); + expect(expectedSha(`${digest} prefixed-${assetName}`, assetName)).toBeNull(); }); - it('reports a checksum mismatch instead of authorizing the archive', () => { + it('rejects a checksum mismatch', async () => { const archive = writeArchive('tampered archive'); const expected = '0'.repeat(64); - const actual = sha256(archive); + const actual = await sha256File(archive); - expect(verifyArchiveChecksum(`${expected} ${assetName}`, assetName, archive)).toEqual({ - status: 'mismatch', - expected, - actual, - }); - }); - - it('treats a suffix-collision entry as an omitted asset', () => { - const archive = writeArchive(); - const digest = sha256(archive); - const sums = `${digest} prefixed-${assetName}`; - - expect(expectedSha(sums, assetName)).toBeNull(); - expect(expectedSha(`SHA256 (prefixed-${assetName}) = ${digest}`, assetName)).toBeNull(); - expect(verifyArchiveChecksum(sums, assetName, archive)).toEqual({ status: 'missing' }); + await expect( + verifyArchiveChecksum(`${expected} ${assetName}`, assetName, archive), + ).resolves.toEqual({ status: 'mismatch', expected, actual }); }); it('rejects a mid-download response error and removes the partial file', async () => { const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-')); tempRoots.push(root); - const dest = path.join(root, 'move-flow.zip'); - const get = vi.fn((_url, onResponse) => { - const request = new EventEmitter(); + const destination = path.join(root, 'move-flow.zip'); + const get = vi.fn((_url, _options, onResponse) => { + const request = new EventEmitter() as ClientRequest; + request.destroy = vi.fn((error?: Error) => { + if (error) request.emit('error', error); + return request; + }); queueMicrotask(() => { - const response = new PassThrough() as DownloadResponse; + const response = new PassThrough() as IncomingMessage; response.statusCode = 200; response.headers = {}; onResponse(response); @@ -114,24 +83,173 @@ describe('install-move-flow', () => { return request; }); - await expect(downloadToFile('https://example.test/move-flow.zip', dest, get)).rejects.toThrow( - 'connection reset during download', - ); - expect(existsSync(dest)).toBe(false); - expect(existsSync(`${dest}.partial`)).toBe(false); + await expect( + downloadToFile('https://example.test/move-flow.zip', destination, 1_000, get), + ).rejects.toThrow('connection reset during download'); + expect(existsSync(destination)).toBe(false); + expect(existsSync(`${destination}.partial`)).toBe(false); }); it('passes PowerShell paths as environment data instead of command source', () => { const archive = `C:\\temp\\move flow's "archive".zip`; - const dest = `C:\\temp\\destination's folder`; - const invocation = powershellExpandArchiveInvocation(archive, dest); + const destination = `C:\\temp\\destination's folder`; + const invocation = powershellExpandArchiveInvocation(archive, destination); const command = invocation.args.join(' '); expect(command).toContain('$env:GITNEXUS_MOVE_FLOW_ARCHIVE_PATH'); expect(command).toContain('$env:GITNEXUS_MOVE_FLOW_DESTINATION_PATH'); expect(command).not.toContain(archive); - expect(command).not.toContain(dest); + expect(command).not.toContain(destination); expect(invocation.env.GITNEXUS_MOVE_FLOW_ARCHIVE_PATH).toBe(archive); - expect(invocation.env.GITNEXUS_MOVE_FLOW_DESTINATION_PATH).toBe(dest); + expect(invocation.env.GITNEXUS_MOVE_FLOW_DESTINATION_PATH).toBe(destination); + }); + + it('uses an artifact-identified versioned user cache', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-home-')); + tempRoots.push(root); + const config = await getMoveFlowInstallConfig({ + ...process.env, + GITNEXUS_HOME: root, + GITNEXUS_MOVE_FLOW_COMPAT: '1', + }); + + expect(config).not.toBeNull(); + expect(config?.installDir.startsWith(path.join(root, 'tools', 'move-flow', '2.0.0'))).toBe( + true, + ); + expect(path.basename(config?.installDir ?? '')).toMatch( + /^(linux-x64|linux-arm64|darwin-arm64|darwin-x64|win32-x64)-[0-9a-f]{12}$/, + ); + if (process.platform === 'linux') expect(config?.releaseTarget).toContain('compat'); + }); + + it('keeps install waiters alive beyond the full download budget', () => { + expect(moveFlowInstallLockWaitMs(30_000)).toBe(120_000); + expect(moveFlowInstallLockWaitMs(90_000)).toBe(240_000); + }); + + it('matches the reported semantic version exactly', () => { + expect(reportedMoveFlowVersion('move-flow 2.0.0')).toBe('2.0.0'); + expect(reportedMoveFlowVersion('move-flow 12.0.0')).not.toBe('2.0.0'); + }); + + it.each([ + ['GITNEXUS_MOVE_FLOW_VERSION', '../escape'], + ['GITNEXUS_MOVE_FLOW_TAG', '../../outside'], + ['GITNEXUS_MOVE_FLOW_REPO', 'owner/repo/extra'], + ])('rejects unsafe release coordinate %s', async (key, value) => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-invalid-config-')); + tempRoots.push(root); + const env = { + ...process.env, + // Cross-platform CI disables automatic provisioning globally; this case + // must still reach release-coordinate validation. + GITNEXUS_SKIP_MOVE_FLOW: '0', + GITNEXUS_MOVE_FLOW_DIR: root, + [key]: value, + }; + + await expect(getMoveFlowInstallConfig(env)).rejects.toThrow('invalid move-flow'); + await expect(installMoveFlow(env)).resolves.toMatchObject({ status: 'failed' }); + expect(existsSync(path.join(root, 'escape'))).toBe(false); + }); + + it.each(['GITNEXUS_SKIP_MOVE_FLOW', 'GITNEXUS_SKIP_OPTIONAL_GRAMMARS'] as const)( + '%s=1 returns a structured skip result without touching the network', + async (flag) => { + const env = { + ...process.env, + GITNEXUS_SKIP_MOVE_FLOW: '0', + GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '0', + [flag]: '1', + }; + await expect(installMoveFlow(env)).resolves.toMatchObject({ + status: 'skipped', + message: expect.stringContaining(flag), + }); + }, + ); + + it('serializes concurrent installers and transfers lock ownership safely', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-lock-')); + tempRoots.push(root); + const lockPath = path.join(root, 'move-flow.install.lock'); + const options = { waitTimeoutMs: 2_000, leaseMs: 500, heartbeatMs: 20, retryMs: 10 }; + const releaseFirst = await acquireInstallLock(lockPath, options); + let secondAcquired = false; + const second = acquireInstallLock(lockPath, options).then((release) => { + secondAcquired = true; + return release; + }); + + await new Promise((resolve) => setTimeout(resolve, 80)); + expect(secondAcquired).toBe(false); + await releaseFirst(); + const releaseSecond = await second; + expect(secondAcquired).toBe(true); + expect(existsSync(lockPath)).toBe(true); + await releaseSecond(); + expect(existsSync(lockPath)).toBe(false); + }); + + it('removes a lock when writing its ownership token fails', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-lock-write-')); + tempRoots.push(root); + const lockPath = path.join(root, 'move-flow.install.lock'); + const handle = { + writeFile: vi.fn(async () => { + throw new Error('simulated disk failure'); + }), + close: vi.fn(async () => {}), + } as unknown as FileHandle; + const removeLock = vi.fn(async (file: string) => { + rmSync(file, { force: true }); + }); + + await expect( + acquireInstallLock( + lockPath, + { waitTimeoutMs: 100 }, + { + openLock: async () => { + writeFileSync(lockPath, ''); + return handle; + }, + removeLock, + }, + ), + ).rejects.toThrow('simulated disk failure'); + expect(handle.close).toHaveBeenCalledOnce(); + expect(removeLock).toHaveBeenCalledWith(lockPath); + expect(existsSync(lockPath)).toBe(false); + }); + + it('reclaims a stable malformed lock after its lease expires', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-lock-stale-')); + tempRoots.push(root); + const lockPath = path.join(root, 'move-flow.install.lock'); + writeFileSync(lockPath, 'not-json'); + const old = new Date(Date.now() - 10_000); + utimesSync(lockPath, old, old); + + const release = await acquireInstallLock(lockPath, { + waitTimeoutMs: 1_000, + leaseMs: 50, + heartbeatMs: 10, + retryMs: 5, + }); + expect(existsSync(lockPath)).toBe(true); + await release(); + expect(existsSync(lockPath)).toBe(false); + }); + + it('runs lock release even when another cleanup task rejects', async () => { + const release = vi.fn(async () => {}); + await expect( + settleMoveFlowCleanup(async () => { + throw new Error('simulated temp cleanup failure'); + }, release), + ).resolves.toBeUndefined(); + expect(release).toHaveBeenCalledOnce(); }); }); diff --git a/gitnexus/test/unit/move/mcp-client.test.ts b/gitnexus/test/unit/move/mcp-client.test.ts index 96710b9e8..27742965d 100644 --- a/gitnexus/test/unit/move/mcp-client.test.ts +++ b/gitnexus/test/unit/move/mcp-client.test.ts @@ -8,13 +8,6 @@ vi.mock('node:child_process', () => ({ execFileSync: vi.fn(), })); -// Keep PATH-resolution tests independent of a developer or CI cache that may -// already contain vendor/move-flow//move-flow. -vi.mock('node:fs', async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, existsSync: vi.fn(() => false) }; -}); - import { MoveFlowMcpClient, MoveFlowToolCallError, @@ -43,13 +36,13 @@ describe('tryCreateMoveFlowClient', () => { }); it('returns MoveFlowMcpClient when binary is found', () => { - mockExecFileSync.mockReturnValue(Buffer.from('')); + mockExecFileSync.mockReturnValue('move-flow 2.0.0'); const client = tryCreateMoveFlowClient(); expect(client).toBeInstanceOf(MoveFlowMcpClient); expect(mockExecFileSync).toHaveBeenCalledWith( 'move-flow', ['--version'], - expect.objectContaining({ stdio: 'ignore' }), + expect.objectContaining({ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }), ); }); @@ -66,14 +59,31 @@ describe('tryCreateMoveFlowClient', () => { 'move-flow;literal-name', ])('passes an explicit binary path directly to execFileSync: %s', (binary) => { process.env.MOVE_FLOW = binary; - mockExecFileSync.mockReturnValue(Buffer.from('')); + mockExecFileSync.mockReturnValue('move-flow 2.0.0'); const client = tryCreateMoveFlowClient(); expect(client).toBeInstanceOf(MoveFlowMcpClient); expect(mockExecFileSync).toHaveBeenCalledWith(binary, ['--version'], { - stdio: 'ignore', + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], timeout: 5000, }); }); + + it.each(['move-flow 1.9.9', 'move-flow 3.0.0', 'unknown build'])( + 'rejects an incompatible version response: %s', + (versionOutput) => { + mockExecFileSync.mockReturnValue(versionOutput); + expect(tryCreateMoveFlowClient('/custom/move-flow')).toBeNull(); + }, + ); + + it.each(['move-flow 2.0.0', 'move-flow 2.1.0-rc1', 'move-flow v2.3.4'])( + 'accepts a compatible-major version response: %s', + (versionOutput) => { + mockExecFileSync.mockReturnValue(versionOutput); + expect(tryCreateMoveFlowClient('/custom/move-flow')).toBeInstanceOf(MoveFlowMcpClient); + }, + ); }); describe('MoveFlowMcpClient', () => { diff --git a/gitnexus/test/unit/move/provision.test.ts b/gitnexus/test/unit/move/provision.test.ts new file mode 100644 index 000000000..13050ad28 --- /dev/null +++ b/gitnexus/test/unit/move/provision.test.ts @@ -0,0 +1,107 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { MoveFlowMcpClient } from '../../../src/core/move/mcp-client.js'; +import { + ensureMoveFlowClient, + type MoveFlowProvisionDependencies, +} from '../../../src/core/move/provision.js'; + +const originalMoveFlow = process.env.MOVE_FLOW; + +afterEach(() => { + if (originalMoveFlow === undefined) delete process.env.MOVE_FLOW; + else process.env.MOVE_FLOW = originalMoveFlow; +}); + +const client = {} as MoveFlowMcpClient; + +describe('ensureMoveFlowClient', () => { + it('does not install when an explicit or PATH binary already resolves', async () => { + const dependencies: MoveFlowProvisionDependencies = { + resolveClient: vi.fn(() => client), + findCached: vi.fn(async () => null), + install: vi.fn(), + }; + + await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBe(client); + expect(dependencies.install).not.toHaveBeenCalled(); + }); + + it('keeps an invalid explicit MOVE_FLOW authoritative', async () => { + process.env.MOVE_FLOW = '/missing/move-flow'; + const dependencies: MoveFlowProvisionDependencies = { + resolveClient: vi.fn(() => null), + findCached: vi.fn(async () => null), + install: vi.fn(), + }; + + await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); + expect(dependencies.install).not.toHaveBeenCalled(); + expect(process.env.MOVE_FLOW).toBe('/missing/move-flow'); + }); + + it('installs once, publishes the cache path to the resolver, and returns a client', async () => { + delete process.env.MOVE_FLOW; + const resolveClient = vi + .fn<(binaryPath?: string) => MoveFlowMcpClient | null>() + .mockReturnValueOnce(null) + .mockReturnValueOnce(client); + const install = vi.fn(async () => ({ + status: 'installed' as const, + binaryPath: '/cache/move-flow', + })); + + await expect( + ensureMoveFlowClient({}, { resolveClient, findCached: async () => null, install }), + ).resolves.toBe(client); + expect(install).toHaveBeenCalledOnce(); + expect(resolveClient).toHaveBeenLastCalledWith('/cache/move-flow'); + expect(process.env.MOVE_FLOW).toBeUndefined(); + }); + + it('soft-fails when installation rejects', async () => { + delete process.env.MOVE_FLOW; + const onLog = vi.fn(); + const dependencies: MoveFlowProvisionDependencies = { + resolveClient: vi.fn(() => null), + findCached: vi.fn(async () => null), + install: vi.fn(async () => { + throw new Error('offline'); + }), + }; + + await expect(ensureMoveFlowClient({ onLog }, dependencies)).resolves.toBeNull(); + expect(onLog).toHaveBeenCalledWith(expect.stringContaining('offline')); + }); + + it('does not retry a failed installation within the same process', async () => { + delete process.env.MOVE_FLOW; + const dependencies: MoveFlowProvisionDependencies = { + resolveClient: vi.fn(() => null), + findCached: vi.fn(async () => null), + install: vi.fn(async () => { + throw new Error('offline'); + }), + }; + + await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); + await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); + expect(dependencies.install).toHaveBeenCalledOnce(); + }); + + it('uses the verified user cache before PATH candidates', async () => { + delete process.env.MOVE_FLOW; + const resolveClient = vi.fn((binaryPath?: string) => + binaryPath === '/cache/move-flow' ? client : null, + ); + const dependencies: MoveFlowProvisionDependencies = { + resolveClient, + findCached: vi.fn(async () => '/cache/move-flow'), + install: vi.fn(), + }; + + await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBe(client); + expect(resolveClient).toHaveBeenCalledTimes(1); + expect(resolveClient).toHaveBeenCalledWith('/cache/move-flow'); + expect(dependencies.install).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/node-table-layout.test.ts b/gitnexus/test/unit/node-table-layout.test.ts new file mode 100644 index 000000000..a4a4451c0 --- /dev/null +++ b/gitnexus/test/unit/node-table-layout.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest'; +import type { GraphNode } from 'gitnexus-shared'; +import { + CONST_SCHEMA, + ENUM_SCHEMA, + ENUM_VARIANT_SCHEMA, + FUNCTION_SCHEMA, + MODULE_SCHEMA, + STRUCT_SCHEMA, +} from '../../src/core/lbug/schema.js'; +import { + getNodeTableCsvHeader, + NODE_TABLE_LAYOUTS, + type LayoutTableName, +} from '../../src/core/lbug/node-table-layout.js'; +import { buildLayoutNodeRow } from '../../src/core/lbug/csv-generator.js'; +import { getCopyQuery } from '../../src/core/lbug/lbug-adapter.js'; + +const schemas: Record = { + Function: FUNCTION_SCHEMA, + Struct: STRUCT_SCHEMA, + Enum: ENUM_SCHEMA, + EnumVariant: ENUM_VARIANT_SCHEMA, + Const: CONST_SCHEMA, + Module: MODULE_SCHEMA, +}; + +const ddlColumns = (ddl: string): string[] => + ddl + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.endsWith(',') && !line.startsWith('PRIMARY KEY')) + .map((line) => line.split(/\s+/, 1)[0]); + +const copyColumns = (query: string): string[] => { + const match = /^COPY\s+`?\w+`?\(([^)]+)\)/.exec(query); + if (!match) throw new Error(`Could not parse COPY query: ${query}`); + return match[1].split(',').map((value) => value.trim()); +}; + +const csvCellCount = (row: string): number => { + let cells = 1; + let quoted = false; + for (let index = 0; index < row.length; index++) { + if (row[index] === '"') { + if (quoted && row[index + 1] === '"') index++; + else quoted = !quoted; + } else if (row[index] === ',' && !quoted) { + cells++; + } + } + return cells; +}; + +describe('shared node-table persistence layouts', () => { + it.each(Object.keys(NODE_TABLE_LAYOUTS) as LayoutTableName[])( + '%s keeps DDL, CSV, row encoding, and COPY order aligned', + (table) => { + const columns = NODE_TABLE_LAYOUTS[table].columns.map(({ name }) => name); + const node = { + id: `${table}:fixture`, + label: table, + properties: { + name: 'fixture', + filePath: 'src/fixture.move', + startLine: 1, + endLine: 2, + language: table === 'Function' ? 'typescript' : 'rust', + }, + } as GraphNode; + + expect(ddlColumns(schemas[table])).toEqual(columns); + expect(getNodeTableCsvHeader(table).split(',')).toEqual(columns); + expect(csvCellCount(buildLayoutNodeRow(table, node, 'fixture content'))).toBe(columns.length); + expect(copyColumns(getCopyQuery(table, '/tmp/fixture.csv'))).toEqual(columns); + }, + ); +}); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 84922760e..3a7890544 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -90,6 +90,7 @@ export default defineConfig({ 'test/integration/extension-binary-real.test.ts', 'test/integration/lbug-delete-nodes-for-files.test.ts', 'test/integration/lbug-query-importers-batch.test.ts', + 'test/integration/move-mixed-language-roundtrip.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', 'test/unit/incremental-orchestration.test.ts', ], @@ -138,6 +139,7 @@ export default defineConfig({ 'test/integration/extension-binary-real.test.ts', 'test/integration/lbug-delete-nodes-for-files.test.ts', 'test/integration/lbug-query-importers-batch.test.ts', + 'test/integration/move-mixed-language-roundtrip.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', 'test/unit/incremental-orchestration.test.ts', ], From 40abd33325e39cadb0bb7067f93de94020c547e7 Mon Sep 17 00:00:00 2001 From: zwxxb Date: Tue, 21 Jul 2026 21:35:34 +0200 Subject: [PATCH 2/4] fix(move): track compiler identity for managed move-flow installs Verify managed installs against release checksums, serialize them with a cross-process lock, and record the compiler identity that produced the Move graph so a compiler change forces a full rebuild. Fail closed when the existing Move graph needs a compiler that is unavailable. --- gitnexus/src/core/move/artifact-download.ts | 226 ++++++++++ gitnexus/src/core/move/constants.ts | 72 +++- gitnexus/src/core/move/discovery.ts | 22 +- gitnexus/src/core/move/install-lock.ts | 128 ++++++ gitnexus/src/core/move/install.ts | 393 ++++++------------ gitnexus/src/core/move/mcp-client.ts | 273 ++++++------ gitnexus/src/core/move/provision.ts | 152 +++++-- gitnexus/src/core/move/release.ts | 7 + gitnexus/src/core/run-analyze.ts | 106 ++++- gitnexus/src/storage/repo-manager.ts | 9 +- gitnexus/test/integration/move-live.test.ts | 12 +- .../move-mixed-language-roundtrip.test.ts | 154 +++---- .../integration/move-run-analyze-e2e.test.ts | 96 +++++ .../unit/incremental-orchestration.test.ts | 46 ++ gitnexus/test/unit/move/discovery.test.ts | 35 ++ .../test/unit/move/incremental-safety.test.ts | 83 +++- .../test/unit/move/install-move-flow.test.ts | 295 ++++++++++++- gitnexus/test/unit/move/mcp-client.test.ts | 101 ++++- gitnexus/test/unit/move/provision.test.ts | 187 ++++++--- gitnexus/vitest.config.ts | 2 + 20 files changed, 1704 insertions(+), 695 deletions(-) create mode 100644 gitnexus/src/core/move/artifact-download.ts create mode 100644 gitnexus/src/core/move/install-lock.ts create mode 100644 gitnexus/test/integration/move-run-analyze-e2e.test.ts create mode 100644 gitnexus/test/unit/move/discovery.test.ts diff --git a/gitnexus/src/core/move/artifact-download.ts b/gitnexus/src/core/move/artifact-download.ts new file mode 100644 index 000000000..a45441050 --- /dev/null +++ b/gitnexus/src/core/move/artifact-download.ts @@ -0,0 +1,226 @@ +import { createHash } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { rename, rm } from 'node:fs/promises'; +import type { ClientRequest, IncomingMessage } from 'node:http'; +import { get as httpsGet, type RequestOptions } from 'node:https'; +import { Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +export type HttpsGet = ( + url: string | URL, + options: RequestOptions, + callback: (response: IncomingMessage) => void, +) => ClientRequest; + +export const MAX_ARCHIVE_BYTES = 256 * 1024 * 1024; + +const MAX_REDIRECTS = 5; +const MAX_ATTEMPTS = 3; +const RETRYABLE_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'EPIPE', + 'ETIMEDOUT', + 'ERR_STREAM_PREMATURE_CLOSE', + 'ENETDOWN', + 'ENETUNREACH', +]); + +class RetryableDownloadError extends Error { + constructor( + message: string, + readonly retryAfterMs = 0, + ) { + super(message); + } +} + +const parseRetryAfter = (value: string | string[] | undefined): number => { + const raw = Array.isArray(value) ? value[0] : value; + if (!raw) return 0; + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const date = Date.parse(raw); + return Number.isFinite(date) ? Math.max(0, date - Date.now()) : 0; +}; + +const wait = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +const safeRequestLabel = (target: string): string => { + const parsed = new URL(target); + return `${parsed.origin}${parsed.pathname}`; +}; + +const isRetryableError = (error: unknown): error is Error => + error instanceof RetryableDownloadError || + (error instanceof Error && RETRYABLE_CODES.has((error as NodeJS.ErrnoException).code ?? '')); + +const downloadAttempt = async ( + initialUrl: string, + partial: string, + deadline: number, + get: HttpsGet, + maxBytes: number, +): Promise => { + if (new URL(initialUrl).protocol !== 'https:') { + throw new Error('move-flow downloads require HTTPS'); + } + let target = initialUrl; + + for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects += 1) { + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error('download timed out'); + + const result = await new Promise<{ redirect?: string }>((resolve, reject) => { + let activeResponse: IncomingMessage | null = null; + let pipelineStarted = false; + const req = get(target, {}, (response) => { + const status = response.statusCode ?? 0; + if (status >= 300 && status < 400 && response.headers.location) { + let redirect: URL; + try { + redirect = new URL(response.headers.location, target); + } catch { + response.destroy(); + reject(new Error(`invalid redirect from ${safeRequestLabel(target)}`)); + return; + } + response.destroy(); + if (redirect.protocol !== 'https:') { + reject(new Error(`refusing non-HTTPS redirect to ${redirect.protocol}`)); + return; + } + resolve({ redirect: redirect.toString() }); + return; + } + + if (status !== 200) { + response.destroy(); + const message = `HTTP ${status} for ${safeRequestLabel(target)}`; + if (status === 408 || status === 429 || status >= 500) { + reject( + new RetryableDownloadError(message, parseRetryAfter(response.headers['retry-after'])), + ); + } else { + reject(new Error(message)); + } + return; + } + + const rawLength = response.headers['content-length']; + const contentLength = Number(Array.isArray(rawLength) ? rawLength[0] : rawLength); + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + response.destroy(); + reject(new Error(`download exceeds ${maxBytes} bytes`)); + return; + } + + let received = 0; + const limit = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + callback( + received > maxBytes ? new Error(`download exceeds ${maxBytes} bytes`) : null, + chunk, + ); + }, + }); + activeResponse = response; + pipelineStarted = true; + void pipeline(response, limit, createWriteStream(partial)).then(() => resolve({}), reject); + }); + const timeout = setTimeout( + () => req.destroy(Object.assign(new Error('download timed out'), { code: 'ETIMEDOUT' })), + Math.max(1, remaining), + ); + req.once('close', () => clearTimeout(timeout)); + req.once('error', (error) => { + if (pipelineStarted) { + // Once the body pipeline owns the response, it must be the only + // operation that settles this attempt. Otherwise cleanup/retry can + // reuse the partial path while the old response is still writing. + activeResponse?.destroy(error); + return; + } + reject(error); + }); + }); + + if (!result.redirect) return; + target = result.redirect; + } + + throw new Error('too many redirects'); +}; + +export const downloadToFile = async ( + url: string, + destination: string, + timeoutMs: number, + get: HttpsGet = httpsGet, + maxBytes = MAX_ARCHIVE_BYTES, +): Promise => { + const partial = `${destination}.partial`; + const deadline = Date.now() + timeoutMs; + + try { + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { + await rm(partial, { force: true }); + try { + await downloadAttempt(url, partial, deadline, get, maxBytes); + await rename(partial, destination); + return; + } catch (error) { + await rm(partial, { force: true }); + if (!isRetryableError(error) || attempt === MAX_ATTEMPTS) throw error; + const retryAfter = + error instanceof RetryableDownloadError ? error.retryAfterMs : attempt * 100; + const remaining = deadline - Date.now(); + if (remaining <= 1) throw error; + await wait(Math.min(Math.max(0, retryAfter), remaining - 1)); + } + } + } finally { + await rm(partial, { force: true }); + } +}; + +export const sha256File = async (file: string): Promise => { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(file)) hash.update(chunk); + return hash.digest('hex'); +}; + +export const expectedSha = (sumsText: string, assetName: string): string | null => { + for (const raw of sumsText.split('\n')) { + const line = raw.trim(); + if (!line) continue; + const standard = /^([0-9a-f]{64})[ \t*]+(\S.*)$/i.exec(line); + if (standard && standard[2] === assetName) return standard[1].toLowerCase(); + const bsd = /^SHA256\s*\((.*)\)\s*=\s*([0-9a-f]{64})$/i.exec(line); + if (bsd && bsd[1] === assetName) return bsd[2].toLowerCase(); + } + return null; +}; + +export type ChecksumVerification = + | { status: 'match'; expected: string; actual: string } + | { status: 'mismatch'; expected: string; actual: string } + | { status: 'missing' }; + +export const verifyArchiveChecksum = async ( + sumsText: string, + assetName: string, + archivePath: string, +): Promise => { + const expected = expectedSha(sumsText, assetName); + if (!expected) return { status: 'missing' }; + const actual = await sha256File(archivePath); + return actual === expected + ? { status: 'match', expected, actual } + : { status: 'mismatch', expected, actual }; +}; diff --git a/gitnexus/src/core/move/constants.ts b/gitnexus/src/core/move/constants.ts index 1801d0943..9b03485df 100644 --- a/gitnexus/src/core/move/constants.ts +++ b/gitnexus/src/core/move/constants.ts @@ -91,13 +91,73 @@ export function isMoveCompilerInputPath(filePath: string): boolean { return normalized.endsWith('.move') || basename === 'Move.toml' || basename === 'Move.lock'; } -/** An available compiler must backfill indexes created while it was absent. */ -export function moveAvailabilityRequiresFullRebuild( - hasMovePackages: boolean, - compilerAvailable: boolean, - indexedWithCompiler: boolean | undefined, +export interface MoveCompilerIdentity { + version: string; + source: 'release' | 'explicit' | 'path'; + fingerprint: string; +} + +/** Dynamic release coordinates used to produce a managed move-flow runtime. */ +export interface MoveFlowReleaseSelection { + version: string; + repository: string; + tag: string; + assetName: string; +} + +function sameMoveFlowReleaseSelection( + left: MoveFlowReleaseSelection | undefined, + right: MoveFlowReleaseSelection | undefined, ): boolean { - return hasMovePackages && compilerAvailable && indexedWithCompiler !== true; + return ( + left !== undefined && + right !== undefined && + left.version === right.version && + left.repository === right.repository && + left.tag === right.tag && + left.assetName === right.assetName + ); +} + +/** Legacy metadata with recorded Move inputs is treated conservatively. */ +export function persistedMoveGraphRequiresCompiler( + moveIngestAvailable: boolean | undefined, + fileHashes: Record | undefined, +): boolean { + if (moveIngestAvailable !== undefined) return moveIngestAvailable; + return Object.keys(fileHashes ?? {}).some(isMoveCompilerInputPath); +} + +/** Decide whether managed provisioning must run before the clean-repo fast path. */ +export function shouldProvisionMoveFlowBeforeFastPath( + hasMovePackages: boolean, + persistedGraphRequiresCompiler: boolean, + indexedCompiler: MoveCompilerIdentity | undefined, + indexedRelease: MoveFlowReleaseSelection | undefined, + desiredRelease: MoveFlowReleaseSelection | undefined, +): boolean { + if (!hasMovePackages) return false; + if (!persistedGraphRequiresCompiler) return true; + if (!indexedCompiler) return true; + if (indexedCompiler.source !== 'release') return false; + return !sameMoveFlowReleaseSelection(indexedRelease, desiredRelease); +} + +/** Compiler-backed facts must be rebuilt when their producer changes. */ +export function moveCompilerRequiresFullRebuild( + hasMovePackages: boolean, + compiler: MoveCompilerIdentity | undefined, + indexedWithCompiler: boolean | undefined, + indexedCompiler: MoveCompilerIdentity | undefined, +): boolean { + if (!hasMovePackages || !compiler) return false; + return ( + indexedWithCompiler !== true || + !indexedCompiler || + indexedCompiler.version !== compiler.version || + indexedCompiler.source !== compiler.source || + indexedCompiler.fingerprint !== compiler.fingerprint + ); } /** diff --git a/gitnexus/src/core/move/discovery.ts b/gitnexus/src/core/move/discovery.ts index 8084473b5..12931558d 100644 --- a/gitnexus/src/core/move/discovery.ts +++ b/gitnexus/src/core/move/discovery.ts @@ -5,22 +5,20 @@ * POSIX-only `find` command, so this works on native Windows. */ -import { glob } from 'glob'; +import { globIterate } from 'glob'; /** * Returns true when the repo contains at least one Move.toml. */ export async function repoHasMove(repoPath: string): Promise { - try { - const matches = await glob(['**/Move.toml'], { - cwd: repoPath, - ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'], - nodir: true, - absolute: false, - dot: false, - }); - return matches.length > 0; - } catch { - return false; + for await (const _match of globIterate(['**/Move.toml'], { + cwd: repoPath, + ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'], + nodir: true, + absolute: false, + dot: false, + })) { + return true; } + return false; } diff --git a/gitnexus/src/core/move/install-lock.ts b/gitnexus/src/core/move/install-lock.ts new file mode 100644 index 000000000..91c98ab8f --- /dev/null +++ b/gitnexus/src/core/move/install-lock.ts @@ -0,0 +1,128 @@ +import { randomUUID } from 'node:crypto'; +import type { Stats } from 'node:fs'; +import { type FileHandle, mkdir, open, readFile, rm, stat, utimes } from 'node:fs/promises'; +import path from 'node:path'; + +export interface LockOptions { + waitTimeoutMs?: number; + leaseMs?: number; + heartbeatMs?: number; + retryMs?: number; +} + +export interface InstallLockIo { + openLock(lockPath: string): Promise; + removeLock(lockPath: string): Promise; +} + +const DEFAULT_LOCK_WAIT_MS = 60_000; +const INSTALL_COMPLETION_GRACE_MS = 60_000; +const DEFAULT_LOCK_LEASE_MS = 60_000; +const DEFAULT_HEARTBEAT_MS = 5_000; +const DEFAULT_LOCK_RETRY_MS = 100; + +const defaultLockIo: InstallLockIo = { + openLock: (lockPath) => open(lockPath, 'wx'), + removeLock: async (lockPath) => { + await rm(lockPath, { force: true }); + }, +}; + +const wait = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +/** Allow for the artifact download, extraction, publication, and version probe. */ +export const moveFlowInstallLockWaitMs = (httpTimeoutMs: number): number => + Math.max(DEFAULT_LOCK_WAIT_MS, httpTimeoutMs * 2 + INSTALL_COMPLETION_GRACE_MS); + +const readLock = async (lockPath: string): Promise => { + try { + const parsed = JSON.parse(await readFile(lockPath, 'utf8')) as { token?: unknown }; + return typeof parsed.token === 'string' ? parsed.token : null; + } catch { + return null; + } +}; + +const statOrNull = async (file: string): Promise => { + try { + return await stat(file); + } catch { + return null; + } +}; + +const removeStaleLock = async (lockPath: string, leaseMs: number): Promise => { + const first = await statOrNull(lockPath); + if (!first || Date.now() - first.mtimeMs <= leaseMs) return; + const token = await readLock(lockPath); + + await wait(25); + const second = await statOrNull(lockPath); + if ( + !second || + second.mtimeMs !== first.mtimeMs || + Date.now() - second.mtimeMs <= leaseMs || + (await readLock(lockPath)) !== token + ) { + return; + } + await rm(lockPath, { force: true }); +}; + +export async function acquireInstallLock( + lockPath: string, + options: LockOptions = {}, + io: InstallLockIo = defaultLockIo, +): Promise<() => Promise> { + const waitTimeoutMs = options.waitTimeoutMs ?? DEFAULT_LOCK_WAIT_MS; + const leaseMs = options.leaseMs ?? DEFAULT_LOCK_LEASE_MS; + const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS; + const retryMs = options.retryMs ?? DEFAULT_LOCK_RETRY_MS; + const deadline = Date.now() + waitTimeoutMs; + const token = randomUUID(); + + await mkdir(path.dirname(lockPath), { recursive: true }); + while (Date.now() < deadline) { + let created = false; + try { + const handle = await io.openLock(lockPath); + created = true; + try { + await handle.writeFile(JSON.stringify({ token, pid: process.pid })); + } finally { + await handle.close(); + } + + let heartbeatRunning = false; + const heartbeat = setInterval(() => { + if (heartbeatRunning) return; + heartbeatRunning = true; + void (async () => { + if ((await readLock(lockPath)) !== token) return; + const now = new Date(); + await utimes(lockPath, now, now); + })() + .catch(() => {}) + .finally(() => { + heartbeatRunning = false; + }); + }, heartbeatMs); + heartbeat.unref(); + + return async () => { + clearInterval(heartbeat); + if ((await readLock(lockPath)) === token) await rm(lockPath, { force: true }); + }; + } catch (error) { + if (created) await io.removeLock(lockPath).catch(() => {}); + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + await removeStaleLock(lockPath, leaseMs); + await wait(retryMs); + } + } + + throw new Error('timed out waiting for another move-flow installation'); +} diff --git a/gitnexus/src/core/move/install.ts b/gitnexus/src/core/move/install.ts index 4d724b932..679a8e72c 100644 --- a/gitnexus/src/core/move/install.ts +++ b/gitnexus/src/core/move/install.ts @@ -1,27 +1,37 @@ -import { createHash, randomUUID } from 'node:crypto'; +import { createHash } from 'node:crypto'; import { execFile } from 'node:child_process'; -import { createReadStream, createWriteStream, type Dirent, type Stats } from 'node:fs'; +import { constants as fsConstants, type Dirent } from 'node:fs'; import { + access, chmod, copyFile, - type FileHandle, + lstat, mkdir, mkdtemp, - open, readFile, readdir, rename, rm, - stat, - utimes, writeFile, } from 'node:fs/promises'; -import type { ClientRequest, IncomingMessage } from 'node:http'; -import { get as httpsGet, type RequestOptions } from 'node:https'; +import { get as httpsGet } from 'node:https'; import os from 'node:os'; import path from 'node:path'; -import { pipeline } from 'node:stream/promises'; -import { MOVE_FLOW_RELEASE } from './release.js'; +import { + downloadToFile, + MAX_ARCHIVE_BYTES, + sha256File, + verifyArchiveChecksum, +} from './artifact-download.js'; +import { acquireInstallLock, moveFlowInstallLockWaitMs } from './install-lock.js'; +import type { MoveFlowReleaseSelection } from './constants.js'; +import { isCompatibleMoveFlowVersion, MOVE_FLOW_RELEASE } from './release.js'; + +export interface VerifiedMoveFlowBinary { + binaryPath: string; + version: string; + fingerprint: string; +} export type MoveFlowInstallStatus = | 'available' @@ -32,8 +42,9 @@ export type MoveFlowInstallStatus = export interface MoveFlowInstallResult { status: MoveFlowInstallStatus; - binaryPath?: string; message?: string; + /** Verified runtime descriptor, set on 'available' and 'installed'. */ + binary?: VerifiedMoveFlowBinary; } export interface MoveFlowInstallConfig { @@ -52,6 +63,7 @@ export interface MoveFlowInstallConfig { } interface MoveFlowCacheMetadata { + schemaVersion: 1; version: string; repository: string; tag: string; @@ -60,64 +72,28 @@ interface MoveFlowCacheMetadata { binarySha256: string; } -export interface LockOptions { - waitTimeoutMs?: number; - leaseMs?: number; - heartbeatMs?: number; - retryMs?: number; -} - -export interface InstallLockIo { - openLock(lockPath: string): Promise; - removeLock(lockPath: string): Promise; -} - interface PowerShellInvocation { args: string[]; env: NodeJS.ProcessEnv; } -type HttpsGet = ( - url: string | URL, - options: RequestOptions, - callback: (response: IncomingMessage) => void, -) => ClientRequest; - const DEFAULT_HTTP_TIMEOUT_MS = 30_000; -const DEFAULT_LOCK_WAIT_MS = 60_000; -const INSTALL_COMPLETION_GRACE_MS = 60_000; -const DEFAULT_LOCK_LEASE_MS = 60_000; -const DEFAULT_HEARTBEAT_MS = 5_000; -const DEFAULT_LOCK_RETRY_MS = 100; - -const defaultLockIo: InstallLockIo = { - openLock: (lockPath) => open(lockPath, 'wx'), - removeLock: async (lockPath) => { - await rm(lockPath, { force: true }); - }, -}; - -const wait = (ms: number): Promise => - new Promise((resolve) => { - setTimeout(resolve, ms); - }); +const MAX_CHECKSUM_BYTES = 1024 * 1024; +const MAX_BINARY_BYTES = 128 * 1024 * 1024; +const MAX_METADATA_BYTES = 64 * 1024; const parsePositiveInteger = (value: string | undefined, fallback: number): number => { const parsed = Number(value); return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; }; -/** - * A waiter must outlive a healthy cold install: two sequential downloads plus - * checksum, extraction, copy, and the executable version probe. - */ -export const moveFlowInstallLockWaitMs = (httpTimeoutMs: number): number => - Math.max(DEFAULT_LOCK_WAIT_MS, httpTimeoutMs * 2 + INSTALL_COMPLETION_GRACE_MS); - const validateReleaseCoordinates = (version: string, repository: string, tag: string): void => { if (!/^\d+\.\d+\.\d+$/.test(version)) { throw new Error(`invalid move-flow version '${version}'; expected X.Y.Z`); } + if (!isCompatibleMoveFlowVersion(version)) { + throw new Error(`unsupported move-flow major version '${version}'`); + } if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) { throw new Error(`invalid move-flow repository '${repository}'; expected owner/name`); } @@ -156,10 +132,9 @@ const execFileOutput = ( ); }); -const linuxNeedsCompatBuild = async (env: NodeJS.ProcessEnv): Promise => { - if (process.platform !== 'linux') return false; - if (env.GITNEXUS_MOVE_FLOW_COMPAT === '1') return true; +let detectedLinuxCompatibility: Promise | undefined; +const detectLinuxCompatibility = async (): Promise => { if (process.arch === 'x64') { try { const cpuinfo = await readFile('/proc/cpuinfo', 'utf8'); @@ -172,15 +147,22 @@ const linuxNeedsCompatBuild = async (env: NodeJS.ProcessEnv): Promise = try { const output = await execFileOutput('ldd', ['--version'], { timeout: 3_000 }); const match = /(\d+)\.(\d+)/.exec(output.split('\n')[0] ?? ''); - if (!match) return false; + if (!match) return true; const major = Number(match[1]); const minor = Number(match[2]); return major < 2 || (major === 2 && minor < 34); } catch { - return false; + return true; } }; +const linuxNeedsCompatBuild = (env: NodeJS.ProcessEnv): Promise => { + if (process.platform !== 'linux') return Promise.resolve(false); + if (env.GITNEXUS_MOVE_FLOW_COMPAT === '1') return Promise.resolve(true); + detectedLinuxCompatibility ??= detectLinuxCompatibility(); + return detectedLinuxCompatibility; +}; + const releaseTargetForPlatform = async ( key: string, env: NodeJS.ProcessEnv, @@ -249,41 +231,20 @@ export async function getMoveFlowInstallConfig( }; } -export const sha256File = async (file: string): Promise => { - const hash = createHash('sha256'); - for await (const chunk of createReadStream(file)) hash.update(chunk); - return hash.digest('hex'); -}; - -export const expectedSha = (sumsText: string, assetName: string): string | null => { - for (const raw of sumsText.split('\n')) { - const line = raw.trim(); - if (!line) continue; - const standard = /^([0-9a-f]{64})[ \t*]+(\S.*)$/i.exec(line); - if (standard && standard[2] === assetName) return standard[1].toLowerCase(); - const bsd = /^SHA256\s*\((.*)\)\s*=\s*([0-9a-f]{64})$/i.exec(line); - if (bsd && bsd[1] === assetName) return bsd[2].toLowerCase(); - } - return null; -}; - -export type ChecksumVerification = - | { status: 'match'; expected: string; actual: string } - | { status: 'mismatch'; expected: string; actual: string } - | { status: 'missing' }; - -export const verifyArchiveChecksum = async ( - sumsText: string, - assetName: string, - archivePath: string, -): Promise => { - const expected = expectedSha(sumsText, assetName); - if (!expected) return { status: 'missing' }; - const actual = await sha256File(archivePath); - return actual === expected - ? { status: 'match', expected, actual } - : { status: 'mismatch', expected, actual }; -}; +/** Resolve the configured release without downloading or starting move-flow. */ +export async function getMoveFlowReleaseSelection( + env: NodeJS.ProcessEnv = process.env, +): Promise { + const config = await getMoveFlowInstallConfig(env); + return config + ? { + version: config.version, + repository: config.repository, + tag: config.tag, + assetName: config.assetName, + } + : undefined; +} export const powershellExpandArchiveInvocation = ( archive: string, @@ -306,159 +267,6 @@ export const powershellExpandArchiveInvocation = ( }; }; -export const downloadToFile = async ( - url: string, - destination: string, - timeoutMs: number, - get: HttpsGet = httpsGet, -): Promise => { - const partial = `${destination}.partial`; - const deadline = Date.now() + timeoutMs; - - const request = (target: string, redirects: number): Promise => - new Promise((resolve, reject) => { - if (redirects > 5) { - reject(new Error('too many redirects')); - return; - } - - const req = get(target, {}, (response) => { - const status = response.statusCode ?? 0; - if (status >= 300 && status < 400 && response.headers.location) { - response.once('error', reject); - response.resume(); - request(new URL(response.headers.location, target).toString(), redirects + 1).then( - resolve, - reject, - ); - return; - } - if (status !== 200) { - response.once('error', reject); - response.resume(); - reject(new Error(`HTTP ${status} for ${target}`)); - return; - } - - pipeline(response, createWriteStream(partial)) - .then(() => rename(partial, destination)) - .then(resolve, reject); - }); - const timeout = setTimeout( - () => req.destroy(new Error('download timed out')), - Math.max(1, deadline - Date.now()), - ); - req.once('close', () => clearTimeout(timeout)); - req.on('error', reject); - }); - - try { - await request(url, 0); - } catch (error) { - await rm(partial, { force: true }); - throw error; - } -}; - -const readLock = async (lockPath: string): Promise => { - try { - const parsed = JSON.parse(await readFile(lockPath, 'utf8')) as { token?: unknown }; - return typeof parsed.token === 'string' ? parsed.token : null; - } catch { - return null; - } -}; - -const statOrNull = async (file: string): Promise => { - try { - return await stat(file); - } catch { - return null; - } -}; - -const removeStaleLock = async (lockPath: string, leaseMs: number): Promise => { - const first = await statOrNull(lockPath); - if (!first || Date.now() - first.mtimeMs <= leaseMs) return; - const token = await readLock(lockPath); - - await wait(25); - const second = await statOrNull(lockPath); - if ( - !second || - second.mtimeMs !== first.mtimeMs || - Date.now() - second.mtimeMs <= leaseMs || - (await readLock(lockPath)) !== token - ) { - return; - } - await rm(lockPath, { force: true }); -}; - -export async function acquireInstallLock( - lockPath: string, - options: LockOptions = {}, - io: InstallLockIo = defaultLockIo, -): Promise<() => Promise> { - const waitTimeoutMs = options.waitTimeoutMs ?? DEFAULT_LOCK_WAIT_MS; - const leaseMs = options.leaseMs ?? DEFAULT_LOCK_LEASE_MS; - const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS; - const retryMs = options.retryMs ?? DEFAULT_LOCK_RETRY_MS; - const deadline = Date.now() + waitTimeoutMs; - const token = randomUUID(); - - await mkdir(path.dirname(lockPath), { recursive: true }); - while (Date.now() < deadline) { - let created = false; - try { - const handle = await io.openLock(lockPath); - created = true; - try { - await handle.writeFile(JSON.stringify({ token, pid: process.pid })); - } finally { - await handle.close(); - } - - let heartbeatRunning = false; - const heartbeat = setInterval(() => { - if (heartbeatRunning) return; - heartbeatRunning = true; - void (async () => { - if ((await readLock(lockPath)) !== token) return; - const now = new Date(); - await utimes(lockPath, now, now); - })() - .catch(() => {}) - .finally(() => { - heartbeatRunning = false; - }); - }, heartbeatMs); - heartbeat.unref(); - - return async () => { - clearInterval(heartbeat); - if ((await readLock(lockPath)) === token) await rm(lockPath, { force: true }); - }; - } catch (error) { - if (created) { - await io.removeLock(lockPath).catch(() => {}); - } - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - await removeStaleLock(lockPath, leaseMs); - await wait(retryMs); - } - } - - throw new Error('timed out waiting for another move-flow installation'); -} - -export async function settleMoveFlowCleanup( - ...tasks: Array<(() => Promise) | undefined> -): Promise { - const active = tasks.filter((task): task is () => Promise => task !== undefined); - await Promise.allSettled(active.map((task) => Promise.resolve().then(task))); -} - export const reportedMoveFlowVersion = (output: string): string | null => /(?:^|\s)v?(\d+\.\d+\.\d+)(?:\s|$)/.exec(output)?.[1] ?? null; @@ -480,33 +288,69 @@ const expectedMetadata = ( assetName: config.assetName, }); -const validCachedInstall = async (config: MoveFlowInstallConfig): Promise => { +const verifiedBinary = ( + config: MoveFlowInstallConfig, + metadata: MoveFlowCacheMetadata, +): VerifiedMoveFlowBinary => ({ + binaryPath: config.binaryPath, + version: metadata.version, + fingerprint: createHash('sha256') + .update( + `${metadata.repository}\0${metadata.tag}\0${metadata.assetName}\0${metadata.binarySha256}`, + ) + .digest('hex'), +}); + +const validCachedInstall = async ( + config: MoveFlowInstallConfig, +): Promise => { try { + const metadataStat = await lstat(config.metadataPath); + if ( + !metadataStat.isFile() || + metadataStat.size <= 0 || + metadataStat.size > MAX_METADATA_BYTES + ) { + return null; + } const metadata = JSON.parse( await readFile(config.metadataPath, 'utf8'), ) as MoveFlowCacheMetadata; const expected = expectedMetadata(config); + const binaryStat = await lstat(config.binaryPath); + if (!binaryStat.isFile() || binaryStat.size <= 0 || binaryStat.size > MAX_BINARY_BYTES) { + return null; + } + await access( + config.binaryPath, + process.platform === 'win32' ? fsConstants.F_OK : fsConstants.X_OK, + ); if ( + metadata.schemaVersion !== 1 || metadata.version !== expected.version || metadata.repository !== expected.repository || metadata.tag !== expected.tag || metadata.assetName !== expected.assetName || metadata.binarySha256 !== (await sha256File(config.binaryPath)) ) { - return false; + return null; } - return exactVersionMatches(config.binaryPath, config.version); + if (!(await exactVersionMatches(config.binaryPath, config.version))) { + return null; + } + return verifiedBinary(config, metadata); } catch { - return false; + return null; } }; -export async function findCachedMoveFlow( +export async function findCachedMoveFlowBinary( env: NodeJS.ProcessEnv = process.env, -): Promise { +): Promise { try { const config = await getMoveFlowInstallConfig(env); - return config && (await validCachedInstall(config)) ? config.binaryPath : null; + if (!config) return null; + return validCachedInstall(config); } catch { return null; } @@ -533,6 +377,7 @@ const extractZip = async (archive: string, destination: string): Promise = const findExtractedBinary = async (root: string, binaryName: string): Promise => { const stack = [root]; const names = new Set([binaryName, 'move-flow']); + const candidates: string[] = []; while (stack.length > 0) { const current = stack.pop(); if (!current) break; @@ -540,10 +385,15 @@ const findExtractedBinary = async (root: string, binaryName: string): Promise 0 && binaryStat.size <= MAX_BINARY_BYTES + ? candidate + : null; }; export async function installMoveFlow( @@ -572,8 +422,9 @@ export async function installMoveFlow( message: `unsupported platform ${process.platform}-${process.arch}; set MOVE_FLOW to provide a binary`, }; } - if (await validCachedInstall(config)) { - return { status: 'available', binaryPath: config.binaryPath }; + const cached = await validCachedInstall(config); + if (cached) { + return { status: 'available', binary: cached }; } let releaseLock: (() => Promise) | undefined; @@ -583,19 +434,28 @@ export async function installMoveFlow( releaseLock = await acquireInstallLock(config.lockPath, { waitTimeoutMs: moveFlowInstallLockWaitMs(config.httpTimeoutMs), }); - if (await validCachedInstall(config)) { - return { status: 'available', binaryPath: config.binaryPath }; + const published = await validCachedInstall(config); + if (published) { + return { status: 'available', binary: published }; } tempDir = await mkdtemp(path.join(os.tmpdir(), 'move-flow-')); const archivePath = path.join(tempDir, config.assetName); const sumsPath = path.join(tempDir, 'SHA256SUMS'); const extractDir = path.join(tempDir, 'extract'); - await downloadToFile(`${config.releaseBase}/SHA256SUMS`, sumsPath, config.httpTimeoutMs); + await downloadToFile( + `${config.releaseBase}/SHA256SUMS`, + sumsPath, + config.httpTimeoutMs, + httpsGet, + MAX_CHECKSUM_BYTES, + ); await downloadToFile( `${config.releaseBase}/${config.assetName}`, archivePath, config.httpTimeoutMs, + httpsGet, + MAX_ARCHIVE_BYTES, ); const verification = await verifyArchiveChecksum( @@ -638,6 +498,7 @@ export async function installMoveFlow( } const metadata: MoveFlowCacheMetadata = { + schemaVersion: 1, ...expectedMetadata(config), archiveSha256: verification.actual, binarySha256: await sha256File(stagedBinary), @@ -647,17 +508,17 @@ export async function installMoveFlow( await rm(config.installDir, { recursive: true, force: true }); await rename(stagedDir, config.installDir); stagedDir = undefined; - return { status: 'installed', binaryPath: config.binaryPath }; + return { status: 'installed', binary: verifiedBinary(config, metadata) }; } catch (error) { return { status: 'failed', message: `installation failed: ${error instanceof Error ? error.message : String(error)}`, }; } finally { - await settleMoveFlowCleanup( - tempDir ? () => rm(tempDir, { recursive: true, force: true }) : undefined, - stagedDir ? () => rm(stagedDir, { recursive: true, force: true }) : undefined, - releaseLock, - ); + await Promise.allSettled([ + tempDir && rm(tempDir, { recursive: true, force: true }), + stagedDir && rm(stagedDir, { recursive: true, force: true }), + releaseLock?.(), + ]); } } diff --git a/gitnexus/src/core/move/mcp-client.ts b/gitnexus/src/core/move/mcp-client.ts index 828621321..8e6bb0cc4 100644 --- a/gitnexus/src/core/move/mcp-client.ts +++ b/gitnexus/src/core/move/mcp-client.ts @@ -9,7 +9,7 @@ import { spawn, execFileSync, type ChildProcessWithoutNullStreams } from 'node:child_process'; import { createInterface } from 'node:readline'; import type { MoveFactsMap, CallGraphMap } from './compiler-facts.js'; -import { MOVE_FLOW_RELEASE } from './release.js'; +import { isCompatibleMoveFlowVersion } from './release.js'; interface JsonRpcResponse { jsonrpc: '2.0'; @@ -193,6 +193,30 @@ export class MoveFlowMcpClient implements MoveFlowClient { this.binaryPath = binaryPath || process.env.MOVE_FLOW || 'move-flow'; } + private failProcess( + proc: ChildProcessWithoutNullStreams, + error: Error, + killProcess = true, + ): void { + if (this.proc !== proc) return; + this.proc = null; + this.initialized = false; + this.initPromise = null; + this.capsPromise = null; + for (const pending of this.pending.values()) { + clearTimeout(pending.timeout); + pending.reject(error); + } + this.pending.clear(); + if (killProcess) { + try { + proc.kill(); + } catch { + /* process may already be dead */ + } + } + } + private async ensureStarted(): Promise { if (this.initialized) return; if (this.initPromise) return this.initPromise; @@ -236,42 +260,10 @@ export class MoveFlowMcpClient implements MoveFlowClient { clearInitTimeout(); if (initId !== null) this.pending.delete(initId); rl?.close(); - - // An old child's late error/exit must not tear down a newer retry. - if (this.proc === proc) { - this.proc = null; - this.initialized = false; - } - if (killProcess) { - try { - proc.kill(); - } catch { - /* process may already be dead */ - } - } + this.failProcess(proc, err, killProcess); reject(err); }; - const failRunningProcess = (err: Error, killProcess = false): void => { - if (this.proc !== proc) return; - for (const [, pending] of this.pending) { - clearTimeout(pending.timeout); - pending.reject(err); - } - this.pending.clear(); - this.initialized = false; - this.initPromise = null; - this.capsPromise = null; - this.proc = null; - if (killProcess) { - try { - proc.kill(); - } catch { - /* process may already be dead */ - } - } - }; - timeout = setTimeout(() => { failInitialization(new Error('move-flow MCP server did not respond within 30s')); }, 30000); @@ -293,6 +285,9 @@ export class MoveFlowMcpClient implements MoveFlowClient { if (this.stderrLines.length > MoveFlowMcpClient.MAX_STDERR) { this.stderrLines.shift(); } + const wrapped = new Error(`move-flow stdin failed: ${err.message}${this.stderrContext()}`); + if (!initSettled) failInitialization(wrapped); + else this.failProcess(proc, wrapped); }); proc.on('error', (err) => { @@ -302,7 +297,7 @@ export class MoveFlowMcpClient implements MoveFlowClient { if (!initSettled) { failInitialization(wrapped); } else { - failRunningProcess(wrapped, true); + this.failProcess(proc, wrapped); } }); @@ -315,13 +310,16 @@ export class MoveFlowMcpClient implements MoveFlowClient { ); return; } - failRunningProcess( + this.failProcess( + proc, new Error(`move-flow exited unexpectedly (code ${code})${this.stderrContext()}`), + false, ); }); rl = createInterface({ input: proc.stdout, crlfDelay: Infinity }); rl.on('line', (line) => { + if (this.proc !== proc) return; if (!line.trim()) return; try { const msg = JSON.parse(line) as JsonRpcResponse; @@ -393,71 +391,66 @@ export class MoveFlowMcpClient implements MoveFlowClient { this.proc.stdin.write(JSON.stringify(msg) + '\n'); } - private async callTool(toolName: string, args: Record): Promise { + private async request( + method: string, + params: unknown, + timeoutMs: number, + timeoutMessage: string, + ): Promise { await this.ensureStarted(); + const proc = this.proc; + if (!proc) throw new Error('move-flow MCP server is not running'); return new Promise((resolve, reject) => { const id = ++this.requestId; - const timeoutMs = resolveMoveFlowToolTimeoutMs(); const timeout = setTimeout(() => { - this.pending.delete(id); - try { - this.proc?.kill(); - } catch { - /* process may already be dead */ - } - reject( - new Error( - `move-flow '${toolName}' timed out after ${timeoutMs}ms ` + - '(raise GITNEXUS_MOVE_FLOW_TIMEOUT_MS for large packages)', - ), - ); + this.failProcess(proc, new Error(timeoutMessage)); }, timeoutMs); - this.pending.set(id, { resolve: (result) => { clearTimeout(timeout); - if (!isMcpCallToolResult(result)) { - resolve(result); - return; - } - // Tool-level failures (e.g. package build failures) arrive as a - // SUCCESS result with `isError: true` and the diagnostic as content - // text - reject rather than hand the error text to callers as data. - if (result.isError === true) { - const text = - typeof result.content?.[0]?.text === 'string' - ? result.content[0].text - : `move-flow '${toolName}' reported an unspecified tool error`; - reject(new MoveFlowToolCallError(text)); - return; - } - if (result.content?.[0]?.text) { - try { - resolve(JSON.parse(result.content[0].text)); - } catch { - resolve(result.content[0].text); - } - } else { - resolve(result); - } + resolve(result); }, - reject: (err) => { + reject: (error) => { clearTimeout(timeout); - reject(err); + reject(error); }, timeout, }); - this.send({ - jsonrpc: '2.0', - id, - method: 'tools/call', - params: { name: toolName, arguments: args }, - }); + try { + this.send({ jsonrpc: '2.0', id, method, params }); + } catch (error) { + this.failProcess(proc, error instanceof Error ? error : new Error(String(error))); + } }); } + private async callTool(toolName: string, args: Record): Promise { + const timeoutMs = resolveMoveFlowToolTimeoutMs(); + const result = await this.request( + 'tools/call', + { name: toolName, arguments: args }, + timeoutMs, + `move-flow '${toolName}' timed out after ${timeoutMs}ms ` + + '(raise GITNEXUS_MOVE_FLOW_TIMEOUT_MS for large packages)', + ); + if (!isMcpCallToolResult(result)) return result; + if (result.isError === true) { + const text = + typeof result.content?.[0]?.text === 'string' + ? result.content[0].text + : `move-flow '${toolName}' reported an unspecified tool error`; + throw new MoveFlowToolCallError(text); + } + if (!result.content?.[0]?.text) return result; + try { + return JSON.parse(result.content[0].text); + } catch { + return result.content[0].text; + } + } + async callGraph(packagePath: string): Promise { return (await this.callTool('move_package_query', { package_path: packagePath, @@ -488,76 +481,37 @@ export class MoveFlowMcpClient implements MoveFlowClient { /** Raw JSON-RPC request (non-`tools/call`), e.g. `tools/list`. */ private async rpcRequest(method: string, params?: unknown): Promise { - await this.ensureStarted(); - return new Promise((resolve, reject) => { - const id = ++this.requestId; - const timeout = setTimeout(() => { - this.pending.delete(id); - reject(new Error(`move-flow '${method}' timed out after 30s`)); - }, 30000); - this.pending.set(id, { - resolve: (result) => { - clearTimeout(timeout); - resolve(result); - }, - reject: (err) => { - clearTimeout(timeout); - reject(err); - }, - timeout, - }); - this.send({ jsonrpc: '2.0', id, method, params }); - }); + return this.request(method, params, 30_000, `move-flow '${method}' timed out after 30s`); } async capabilities(): Promise { if (this.capsPromise) return this.capsPromise; - this.capsPromise = (async () => { - try { - const listed = await this.rpcRequest('tools/list', {}); - const tools = ( - isMcpListToolResult(listed) ? (listed.tools ?? []) : [] - ) as MoveFlowToolInfo[]; - return detectMoveFlowCapabilities(tools); - } catch { - // Probe failure → facts unavailable; the ingest phase trips its hard gate. - return { hasFactsQuery: false, hasStatusTool: false }; - } - })(); - return this.capsPromise; + const probe = this.rpcRequest('tools/list', {}).then((listed) => { + const tools = (isMcpListToolResult(listed) ? (listed.tools ?? []) : []) as MoveFlowToolInfo[]; + return detectMoveFlowCapabilities(tools); + }); + this.capsPromise = probe; + void probe.catch(() => { + if (this.capsPromise === probe) this.capsPromise = null; + }); + return probe; } async shutdown(): Promise { - const shutdownError = new Error('move-flow client shutdown'); - for (const [, p] of this.pending) { - clearTimeout(p.timeout); - p.reject(shutdownError); + const proc = this.proc; + if (proc) { + proc.stdin?.end(); + this.failProcess(proc, new Error('move-flow client shutdown')); + } else { + this.initialized = false; + this.initPromise = null; + this.capsPromise = null; } - this.pending.clear(); - if (this.proc) { - this.proc.stdin?.end(); - this.proc.kill(); - this.proc = null; - } - this.initialized = false; - this.initPromise = null; - this.capsPromise = null; this.stderrLines.length = 0; } } -/** - * Try to create a MoveFlowMcpClient. Returns null if move-flow binary - * is not found on the system. When `binaryPath` is provided, only that - * provisioned path is probed; the normal resolution order is bypassed. - * - * Resolution order: - * 1. `$MOVE_FLOW` (explicit override for power users / CI). - * 2. `move-flow` on `$PATH` (host install). - */ -const MOVE_FLOW_COMPATIBLE_MAJOR = Number(MOVE_FLOW_RELEASE.version.split('.')[0]); - -function probeBinary(binary: string): boolean { +function probeBinary(binary: string): string | null { try { const output = execFileSync(binary, ['--version'], { encoding: 'utf8', @@ -566,23 +520,30 @@ function probeBinary(binary: string): boolean { }); // Prerelease/build suffixes ("2.1.0-rc1") are accepted; only the major // version gates protocol compatibility, and it follows the release pin. - const version = /(?:^|\s)v?(\d+)\.(\d+)\.(\d+)(?:[-+][0-9A-Za-z.-]+)?(?:\s|$)/.exec(output); - return version !== null && Number(version[1]) === MOVE_FLOW_COMPATIBLE_MAJOR; + const version = /(?:^|\s)v?((\d+)\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)(?:\s|$)/.exec(output); + return version !== null && isCompatibleMoveFlowVersion(version[1]) ? version[1] : null; } catch { - return false; + return null; } } -export function tryCreateMoveFlowClient(binaryPath?: string): MoveFlowMcpClient | null { - if (binaryPath) { - return probeBinary(binaryPath) ? new MoveFlowMcpClient(binaryPath) : null; - } - - const explicit = process.env.MOVE_FLOW; - if (explicit) { - return probeBinary(explicit) ? new MoveFlowMcpClient(explicit) : null; - } - - const onPath = 'move-flow'; - return probeBinary(onPath) ? new MoveFlowMcpClient(onPath) : null; +export interface ResolvedMoveFlowClient { + client: MoveFlowMcpClient; + version: string; +} + +/** Construct a client for a binary that was already verified by the installer. */ +export const createMoveFlowClient = (binaryPath: string): MoveFlowMcpClient => + new MoveFlowMcpClient(binaryPath); + +/** + * Probe and resolve a move-flow candidate exactly once, returning null when + * the binary is missing or reports an incompatible major version. When + * `binaryPath` is provided only that path is probed; otherwise resolution is + * `$MOVE_FLOW` (explicit override), then `move-flow` on `$PATH`. + */ +export function tryResolveMoveFlowClient(binaryPath?: string): ResolvedMoveFlowClient | null { + const binary = binaryPath || process.env.MOVE_FLOW || 'move-flow'; + const version = probeBinary(binary); + return version ? { client: createMoveFlowClient(binary), version } : null; } diff --git a/gitnexus/src/core/move/provision.ts b/gitnexus/src/core/move/provision.ts index a80fe7c64..2cba7007a 100644 --- a/gitnexus/src/core/move/provision.ts +++ b/gitnexus/src/core/move/provision.ts @@ -1,88 +1,152 @@ -import { MoveFlowMcpClient, tryCreateMoveFlowClient } from './mcp-client.js'; -import { findCachedMoveFlow, installMoveFlow, type MoveFlowInstallResult } from './install.js'; -import { MOVE_FLOW_RELEASE } from './release.js'; +import { createHash } from 'node:crypto'; +import type { MoveCompilerIdentity } from './constants.js'; +import { + createMoveFlowClient, + tryResolveMoveFlowClient, + type MoveFlowMcpClient, + type ResolvedMoveFlowClient, +} from './mcp-client.js'; +import { + findCachedMoveFlowBinary, + installMoveFlow, + type MoveFlowInstallResult, + type VerifiedMoveFlowBinary, +} from './install.js'; +import { isCompatibleMoveFlowVersion, MOVE_FLOW_RELEASE } from './release.js'; -// One install attempt per dependency set per process: concurrent callers share -// the in-flight promise, and a settled failure is not retried until restart -// (an explicit MOVE_FLOW or a PATH/cache binary is still re-probed every call). const installAttempts = new WeakMap< MoveFlowProvisionDependencies, Promise >(); +export interface ProvisionedMoveFlow { + client: MoveFlowMcpClient; + identity: MoveCompilerIdentity; +} + export interface MoveFlowProvisionOptions { onLog?: (message: string) => void; + install?: boolean; } export interface MoveFlowProvisionDependencies { - resolveClient: (binaryPath?: string) => MoveFlowMcpClient | null; - findCached: () => Promise; + resolveClient: (binaryPath?: string) => ResolvedMoveFlowClient | null; + createClient: (binaryPath: string) => MoveFlowMcpClient; + findCached: () => Promise; install: () => Promise; } const defaultDependencies: MoveFlowProvisionDependencies = { - resolveClient: tryCreateMoveFlowClient, - findCached: findCachedMoveFlow, + resolveClient: tryResolveMoveFlowClient, + createClient: createMoveFlowClient, + findCached: findCachedMoveFlowBinary, install: installMoveFlow, }; -/** - * Resolve move-flow, provisioning the pinned release only when a caller has - * already detected Move code. Explicit MOVE_FLOW overrides remain authoritative: - * an invalid override is reported instead of silently replaced. - */ -export async function ensureMoveFlowClient( +const localIdentity = ( + source: 'explicit' | 'path', + locator: string, + version: string, +): MoveCompilerIdentity => ({ + version, + source, + fingerprint: createHash('sha256').update(`${source}\0${locator}\0${version}`).digest('hex'), +}); + +const verifiedRuntime = ( + binary: VerifiedMoveFlowBinary, + dependencies: MoveFlowProvisionDependencies, +): ProvisionedMoveFlow | null => + isCompatibleMoveFlowVersion(binary.version) + ? { + client: dependencies.createClient(binary.binaryPath), + identity: { + version: binary.version, + source: 'release', + fingerprint: binary.fingerprint, + }, + } + : null; + +const getInstallAttempt = ( + dependencies: MoveFlowProvisionDependencies, +): Promise => { + const active = installAttempts.get(dependencies); + if (active) return active; + + const created = Promise.resolve().then(dependencies.install); + installAttempts.set(dependencies, created); + const clear = (): void => { + if (installAttempts.get(dependencies) === created) installAttempts.delete(dependencies); + }; + void created.then(clear, clear); + return created; +}; + +/** Resolve move-flow after the caller has already detected Move code. */ +export async function ensureMoveFlowRuntime( options: MoveFlowProvisionOptions = {}, dependencies: MoveFlowProvisionDependencies = defaultDependencies, -): Promise { - if (process.env.MOVE_FLOW) { - const explicit = dependencies.resolveClient(process.env.MOVE_FLOW); - if (explicit) return explicit; +): Promise { + const explicitPath = process.env.MOVE_FLOW; + if (explicitPath) { + const resolved = dependencies.resolveClient(explicitPath); + if (resolved) { + return { + client: resolved.client, + identity: localIdentity('explicit', explicitPath, resolved.version), + }; + } options.onLog?.( - `MOVE_FLOW points to an unavailable binary (${process.env.MOVE_FLOW}); automatic installation was skipped.`, + `MOVE_FLOW points to an unavailable binary (${explicitPath}); automatic installation was skipped.`, ); return null; } - const cachedPath = await dependencies.findCached(); - if (cachedPath) { - const cached = dependencies.resolveClient(cachedPath); - if (cached) return cached; + const cached = await dependencies.findCached(); + if (cached) { + const runtime = verifiedRuntime(cached, dependencies); + if (runtime) return runtime; } const existing = dependencies.resolveClient(); - if (existing) return existing; - - let attempt = installAttempts.get(dependencies); - if (!attempt) { - options.onLog?.( - `Move code detected; ensuring move-flow ${MOVE_FLOW_RELEASE.version} is available.`, - ); - attempt = dependencies.install(); - installAttempts.set(dependencies, attempt); + if (existing) { + // The locator must be stable across shells: $PATH itself differs between + // terminals/CI steps, and a fingerprint churn forces a full re-index. + return { + client: existing.client, + identity: localIdentity('path', 'move-flow', existing.version), + }; } + if (options.install === false) return null; + + options.onLog?.( + `Move code detected; ensuring move-flow ${MOVE_FLOW_RELEASE.version} is available.`, + ); let result: MoveFlowInstallResult; try { - result = await attempt; + result = await getInstallAttempt(dependencies); } catch (err) { options.onLog?.( `move-flow installation failed: ${err instanceof Error ? err.message : String(err)}; ` + - '.move files will not be indexed (not retried in this process).', + '.move files will not be indexed.', ); return null; } - if (!result.binaryPath) { + if (!result.binary) { options.onLog?.( - `move-flow is unavailable${result.message ? `: ${result.message}` : ''}; .move files will not be indexed.`, + `move-flow is unavailable${result.message ? `: ${result.message}` : ''}; ` + + '.move files will not be indexed.', ); return null; } - - const installed = dependencies.resolveClient(result.binaryPath); - if (installed) return installed; - - options.onLog?.(`Installed move-flow failed its runtime probe at ${result.binaryPath}.`); - return null; + const runtime = verifiedRuntime(result.binary, dependencies); + if (!runtime) { + options.onLog?.( + `move-flow ${result.binary.version} is protocol-incompatible; .move files will not be indexed.`, + ); + } + return runtime; } diff --git a/gitnexus/src/core/move/release.ts b/gitnexus/src/core/move/release.ts index 1aa9dd419..49605eba4 100644 --- a/gitnexus/src/core/move/release.ts +++ b/gitnexus/src/core/move/release.ts @@ -3,3 +3,10 @@ export const MOVE_FLOW_RELEASE = { repository: 'aptos-labs/aptos-ai', tagPrefix: 'move-flow-v', } as const; + +const COMPATIBLE_MAJOR = Number(MOVE_FLOW_RELEASE.version.split('.')[0]); + +export const isCompatibleMoveFlowVersion = (version: string): boolean => { + const match = /^(\d+)\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.exec(version); + return match !== null && Number(match[1]) === COMPATIBLE_MAJOR; +}; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index c1c421594..02a157219 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -13,10 +13,16 @@ import path from 'path'; import fs from 'fs/promises'; import { execFileSync } from 'child_process'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; -import { isMoveCompilerInputPath, moveAvailabilityRequiresFullRebuild } from './move/constants.js'; +import { + isMoveCompilerInputPath, + moveCompilerRequiresFullRebuild, + persistedMoveGraphRequiresCompiler, + shouldProvisionMoveFlowBeforeFastPath, +} from './move/constants.js'; import { createMoveIngestPhase } from './move/move-ingest.js'; import { repoHasMove } from './move/discovery.js'; -import { ensureMoveFlowClient } from './move/provision.js'; +import { getMoveFlowReleaseSelection } from './move/install.js'; +import { ensureMoveFlowRuntime } from './move/provision.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { initLbug, @@ -941,22 +947,52 @@ export async function runFullAnalysis( options = { ...options, force: true }; } - // Probe before the same-commit fast path: an index built while move-flow was - // unavailable must be backfilled once the compiler becomes available. + // Resolve local runtimes before the same-commit fast path. Managed + // provisioning is needed for a degraded/legacy graph and whenever the + // configured dynamic release differs from the one that built the index. const hasMovePackages = await repoHasMove(repoPath); - const moveFlowClient = hasMovePackages ? await ensureMoveFlowClient({ onLog: log }) : null; - const moveIngestAvailable = hasMovePackages ? moveFlowClient !== null : undefined; - if ( - existingMeta && - moveAvailabilityRequiresFullRebuild( - hasMovePackages, - moveFlowClient !== null, - existingMeta.moveIngestAvailable, - ) - ) { - log('Move compiler became available; forcing a full rebuild to backfill Move symbols.'); + const desiredMoveFlowRelease = hasMovePackages ? await getMoveFlowReleaseSelection() : undefined; + const persistedMoveCompilerRequired = persistedMoveGraphRequiresCompiler( + existingMeta?.moveIngestAvailable, + existingMeta?.fileHashes, + ); + const provisionBeforeFastPath = shouldProvisionMoveFlowBeforeFastPath( + hasMovePackages, + persistedMoveCompilerRequired, + existingMeta?.moveCompilerIdentity, + existingMeta?.moveFlowReleaseSelection, + desiredMoveFlowRelease, + ); + let moveFlow = hasMovePackages + ? await ensureMoveFlowRuntime({ + onLog: log, + install: provisionBeforeFastPath, + }) + : null; + let moveCompilerRebuildApplied = false; + const applyMoveCompilerRebuildPolicy = (): void => { + if ( + !existingMeta || + !moveCompilerRequiresFullRebuild( + hasMovePackages, + moveFlow?.identity, + existingMeta.moveIngestAvailable, + existingMeta.moveCompilerIdentity, + ) + ) { + return; + } + if (!moveCompilerRebuildApplied) { + log( + existingMeta.moveIngestAvailable === true + ? 'Move compiler changed; forcing a full rebuild so persisted facts match it.' + : 'Move compiler became available; forcing a full rebuild to backfill Move symbols.', + ); + } + moveCompilerRebuildApplied = true; options = { ...options, force: true }; - } + }; + applyMoveCompilerRebuildPolicy(); // ── Early-return: already up to date ────────────────────────────── if ( @@ -1058,7 +1094,7 @@ export async function runFullAnalysis( } } await ensureGitNexusIgnored(repoPath); - await moveFlowClient?.shutdown(); + await moveFlow?.client.shutdown(); return { // `resolveRepoIdentityRoot` collapses worktree roots to the // canonical repo basename (#1259) but leaves arbitrary subdirs @@ -1076,6 +1112,23 @@ export async function runFullAnalysis( } } + // From this point onward analysis may write or derive graph data. Never run + // that plan without the compiler that produced an existing Move graph: a + // partial rebuild would silently erase Move rows or degrade global clusters. + if (hasMovePackages && !moveFlow) { + moveFlow = await ensureMoveFlowRuntime({ onLog: log }); + if (persistedMoveCompilerRequired && !moveFlow) { + throw new Error( + 'move-flow is unavailable; the existing Move graph was left unchanged. ' + + 'Restore move-flow or set MOVE_FLOW, then retry analysis.', + ); + } + } + const moveFlowClient = moveFlow?.client ?? null; + const moveIngestAvailable = hasMovePackages ? moveFlow !== null : undefined; + + applyMoveCompilerRebuildPolicy(); + // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: // --embeddings -> load cache, restore, then generate any new ones @@ -2050,6 +2103,13 @@ export async function runFullAnalysis( const newFileHashesRecord: Record = {}; for (const [k, v] of newFileHashes) newFileHashesRecord[k] = v; + // Incremental runs never rewrite Move facts (changed Move compiler inputs + // force a full rebuild), so when the compiler is transiently unavailable + // the previous record still describes the persisted facts. Stamping + // false/undefined here would force a needless full rebuild the moment the + // compiler returns. + const preserveMoveMeta = hasMovePackages && !moveFlow && isIncremental; + // Annotated so the capabilities stamp below is compile-checked against // RepoMeta's status unions (tri-review 4669518496 P1/U3) — an unannotated // literal widens the vectorSearch.status ternary to `string` and the @@ -2108,7 +2168,17 @@ export async function runFullAnalysis( // absence, so this is never conditionally omitted. cjkSegmentation: getSearchFTSCjkSegmentation(), fileHashes: hasGitDir(repoPath) ? newFileHashesRecord : undefined, - moveIngestAvailable, + moveIngestAvailable: preserveMoveMeta + ? existingMeta?.moveIngestAvailable + : moveIngestAvailable, + moveCompilerIdentity: preserveMoveMeta + ? existingMeta?.moveCompilerIdentity + : moveFlow?.identity, + moveFlowReleaseSelection: preserveMoveMeta + ? existingMeta?.moveFlowReleaseSelection + : moveFlow?.identity.source === 'release' + ? desiredMoveFlowRelease + : undefined, // This branch's full live chunk-key set (#2106 R6). `usedKeys` is every // chunk hash touched in this scan — cache HITS included (see parse-impl // usedKeys.add) — so it's complete even on an incremental run. Persisted diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 088fdadc6..bdf3022e6 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -22,6 +22,7 @@ import { randomBytes } from 'crypto'; import { getInferredRepoName, resolveRepoIdentityRoot } from './git.js'; import { retryRename } from './fs-atomic.js'; import { logger } from '../core/logger.js'; +import type { MoveCompilerIdentity, MoveFlowReleaseSelection } from '../core/move/constants.js'; import { branchSlug, BRANCHES_DIR, @@ -162,10 +163,14 @@ export interface RepoMeta { */ fileHashes?: Record; /** - * Whether compiler-backed Move ingestion was available for this index. - * Absent on legacy metadata and non-Move repositories. + * Whether the persisted Move facts are compiler-backed. Absent on legacy + * metadata and non-Move repositories. */ moveIngestAvailable?: boolean; + /** Compiler identity that produced the persisted Move facts. */ + moveCompilerIdentity?: MoveCompilerIdentity; + /** Managed release coordinates, kept separate from the binary fingerprint. */ + moveFlowReleaseSelection?: MoveFlowReleaseSelection; /** * Crash-recovery dirty flag — a generic marker written to the metadata * file (gitnexus.json + its meta.json mirror) BEFORE any destructive DB diff --git a/gitnexus/test/integration/move-live.test.ts b/gitnexus/test/integration/move-live.test.ts index 78b36b276..ad8375ee2 100644 --- a/gitnexus/test/integration/move-live.test.ts +++ b/gitnexus/test/integration/move-live.test.ts @@ -13,15 +13,19 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; -import { tryCreateMoveFlowClient } from '../../src/core/move/mcp-client.js'; +import { tryResolveMoveFlowClient } from '../../src/core/move/mcp-client.js'; import { createMoveIngestPhase } from '../../src/core/move/move-ingest.js'; -import { ensureMoveFlowClient } from '../../src/core/move/provision.js'; +import { ensureMoveFlowRuntime } from '../../src/core/move/provision.js'; import { runMoveIngestPhase } from '../helpers/move-ingest-harness.js'; const requireMoveFlow = process.env.GITNEXUS_REQUIRE_MOVE_FLOW === '1'; const client = requireMoveFlow - ? await ensureMoveFlowClient({ onLog: (message) => console.info(`[move-live] ${message}`) }) - : tryCreateMoveFlowClient(); + ? ( + await ensureMoveFlowRuntime({ + onLog: (message) => console.info(`[move-live] ${message}`), + }) + )?.client + : tryResolveMoveFlowClient()?.client; if (requireMoveFlow && !client) { throw new Error('GITNEXUS_REQUIRE_MOVE_FLOW=1 but MoveFlow provisioning failed'); } diff --git a/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts b/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts index 4738634a8..0df3df497 100644 --- a/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts +++ b/gitnexus/test/integration/move-mixed-language-roundtrip.test.ts @@ -1,53 +1,57 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; +import type { NodeLabel } from '../../src/core/graph/types.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; import { buildTestGraph } from '../helpers/test-graph.js'; +const sharedTables = [ + { + table: 'Function', + otherLanguage: 'typescript', + detailProperty: 'isEntry', + moveDetail: true, + }, + { table: 'Struct', otherLanguage: 'rust', detailProperty: 'isResource', moveDetail: true }, + { table: 'Enum', otherLanguage: 'rust', detailProperty: 'isEvent', moveDetail: true }, + { + table: 'EnumVariant', + otherLanguage: 'rust', + detailProperty: 'parentEnum', + moveDetail: '0x1::sample::Choice', + }, + { table: 'Const', otherLanguage: 'typescript', detailProperty: 'isErrorCode', moveDetail: true }, + { table: 'Module', otherLanguage: 'python', detailProperty: 'moduleAddress', moveDetail: '0x1' }, +] as const; + withTestLbugDB( 'move-mixed-language-roundtrip', () => { describe('mixed-language persistence', () => { - it('stores Move facts and safe defaults for non-Move rows in shared tables', async () => { + it('stores Move facts and safe defaults in every shared table', async () => { const { executeQuery } = await import('../../src/core/lbug/lbug-adapter.js'); - const functions = (await executeQuery( - 'MATCH (f:Function) RETURN f.id AS id, f.language AS language, ' + - 'f.qualifiedName AS qualifiedName, f.isEntry AS isEntry ORDER BY f.id', - )) as Array>; - expect(functions).toEqual([ - { - id: 'Function:move', - language: 'move', - qualifiedName: '0x1::coin::mint', - isEntry: true, - }, - { - id: 'Function:typescript', - language: 'typescript', - qualifiedName: null, - isEntry: false, - }, - ]); - - const structs = (await executeQuery( - 'MATCH (s:`Struct`) RETURN s.id AS id, s.language AS language, ' + - 's.qualifiedName AS qualifiedName, s.isResource AS isResource ORDER BY s.id', - )) as Array>; - expect(structs).toEqual([ - { - id: 'Struct:move', - language: 'move', - qualifiedName: '0x1::coin::CoinStore', - isResource: true, - }, - { - id: 'Struct:rust', - language: 'rust', - qualifiedName: null, - isResource: false, - }, - ]); + for (const testCase of sharedTables) { + const rows = (await executeQuery( + `MATCH (n:\`${testCase.table}\`) ` + + `RETURN n.id AS id, n.language AS language, n.qualifiedName AS qualifiedName, ` + + `n.${testCase.detailProperty} AS detail ORDER BY n.id`, + )) as Array>; + expect(rows).toEqual([ + { + id: `${testCase.table}:move`, + language: 'move', + qualifiedName: `0x1::sample::${testCase.table}`, + detail: testCase.moveDetail, + }, + { + id: `${testCase.table}:${testCase.otherLanguage}`, + language: testCase.otherLanguage, + qualifiedName: null, + detail: typeof testCase.moveDetail === 'boolean' ? false : null, + }, + ]); + } }); }); }, @@ -57,56 +61,34 @@ withTestLbugDB( const storagePath = path.join(path.dirname(dbPath), 'storage'); await fs.mkdir(repoPath, { recursive: true }); - const graph = buildTestGraph([ - { - id: 'Function:move', - label: 'Function', - name: 'mint', - filePath: 'sources/coin.move', - startLine: 1, - endLine: 4, - isExported: true, - extra: { - language: 'move', - qualifiedName: '0x1::coin::mint', - moduleQualifiedName: '0x1::coin', - isEntry: true, + const graph = buildTestGraph( + sharedTables.flatMap((testCase) => [ + { + id: `${testCase.table}:move`, + label: testCase.table as NodeLabel, + name: testCase.table, + filePath: 'sources/sample.move', + startLine: 1, + endLine: 2, + isExported: testCase.table === 'Function', + extra: { + language: 'move', + qualifiedName: `0x1::sample::${testCase.table}`, + [testCase.detailProperty]: testCase.moveDetail, + }, }, - }, - { - id: 'Function:typescript', - label: 'Function', - name: 'mint', - filePath: 'src/coin.ts', - startLine: 1, - endLine: 2, - isExported: true, - extra: { language: 'typescript' }, - }, - { - id: 'Struct:move', - label: 'Struct', - name: 'CoinStore', - filePath: 'sources/coin.move', - startLine: 6, - endLine: 10, - extra: { - language: 'move', - qualifiedName: '0x1::coin::CoinStore', - moduleQualifiedName: '0x1::coin', - isResource: true, + { + id: `${testCase.table}:${testCase.otherLanguage}`, + label: testCase.table as NodeLabel, + name: testCase.table, + filePath: `src/sample.${testCase.otherLanguage}`, + startLine: 1, + endLine: 2, + isExported: testCase.table === 'Function', + extra: { language: testCase.otherLanguage }, }, - }, - { - id: 'Struct:rust', - label: 'Struct', - name: 'CoinStore', - filePath: 'src/coin.rs', - startLine: 1, - endLine: 3, - extra: { language: 'rust' }, - }, - ]); + ]), + ); const { loadGraphToLbug } = await import('../../src/core/lbug/lbug-adapter.js'); await loadGraphToLbug(graph, repoPath, storagePath); diff --git a/gitnexus/test/integration/move-run-analyze-e2e.test.ts b/gitnexus/test/integration/move-run-analyze-e2e.test.ts new file mode 100644 index 000000000..11265977b --- /dev/null +++ b/gitnexus/test/integration/move-run-analyze-e2e.test.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { tryResolveMoveFlowClient } from '../../src/core/move/mcp-client.js'; +import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const requireMoveFlow = process.env.GITNEXUS_REQUIRE_MOVE_FLOW === '1'; +const skipMoveFlow = process.env.GITNEXUS_SKIP_MOVE_FLOW === '1'; +const canRun = requireMoveFlow || (!skipMoveFlow && tryResolveMoveFlowClient() !== null); + +describe.skipIf(!canRun)('Move runFullAnalysis persistence', () => { + it('provisions, persists, and queries a real mixed repository', async () => { + const repo = await createTempDir('gitnexus-move-run-analyze-'); + const home = await createTempDir('gitnexus-move-run-analyze-home-'); + const previousHome = process.env.GITNEXUS_HOME; + const previousExtensionPolicy = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; + try { + process.env.GITNEXUS_HOME = home.dbPath; + process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; + await fs.mkdir(path.join(repo.dbPath, 'sources'), { recursive: true }); + await fs.writeFile( + path.join(repo.dbPath, 'Move.toml'), + '[package]\nname = "live_e2e"\nversion = "1.0.0"\n\n[addresses]\nlive = "0x42"\n', + ); + await fs.writeFile( + path.join(repo.dbPath, 'sources', 'main.move'), + 'module live::main { public entry fun start(_account: &signer) {} }\n', + ); + await fs.writeFile( + path.join(repo.dbPath, 'helper.ts'), + 'export function helper(): number { return 1; }\n', + ); + + await runFullAnalysis( + repo.dbPath, + { + force: true, + skipGit: true, + skipAgentsMd: true, + skipSkills: true, + noStats: true, + workerPoolSize: 2, + }, + { onProgress: () => {} }, + ); + + const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); + const meta = await loadMeta(storagePath); + expect(meta?.moveIngestAvailable).toBe(true); + expect(meta?.moveCompilerIdentity?.version).toMatch(/^2\./); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(lbugPath); + try { + await expect( + adapter.executeQuery( + "MATCH (f:Function) WHERE f.qualifiedName = '0x42::main::start' " + + 'RETURN f.name AS name, f.isEntry AS isEntry', + ), + ).resolves.toEqual([{ name: 'start', isEntry: true }]); + await expect( + adapter.executeQuery( + "MATCH (f:Function) WHERE f.name = 'helper' " + + 'RETURN f.language AS language, f.qualifiedName AS qualifiedName', + ), + ).resolves.toEqual([{ language: 'typescript', qualifiedName: null }]); + await expect( + adapter.executeQuery( + "MATCH (m:Module)-[r:CodeRelation]->(f:Function) WHERE r.type = 'DEFINES' " + + "AND f.qualifiedName = '0x42::main::start' " + + 'RETURN m.qualifiedName AS module, r.type AS type', + ), + ).resolves.toEqual([{ module: '0x42::main', type: 'DEFINES' }]); + await expect( + adapter.executeQuery( + 'MATCH (n) WITH n.id AS id, count(n) AS copies ' + 'WHERE copies > 1 RETURN id, copies', + ), + ).resolves.toEqual([]); + } finally { + await adapter.closeLbug(); + } + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + if (previousExtensionPolicy === undefined) { + delete process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; + } else { + process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = previousExtensionPolicy; + } + await repo.cleanup(); + await home.cleanup(); + } + }, 180_000); +}); diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index ba99493c3..0b5473099 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -191,6 +191,52 @@ describe('runFullAnalysis — incremental orchestration', () => { } }, 300_000); + it('leaves a compiler-backed Move index untouched when move-flow is unavailable', async () => { + const repo = await setupMiniRepo(); + try { + // An explicit MOVE_FLOW override that fails its probe is authoritative + // and skips auto-install — deterministic "compiler unavailable" offline. + vi.stubEnv('MOVE_FLOW', path.join(repo.dbPath, 'nonexistent-move-flow')); + await writeFile( + path.join(repo.dbPath, 'Move.toml'), + '[package]\nname = "mini"\nversion = "1.0.0"\n', + ); + gitCommitAll(repo.dbPath, 'add move package'); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }); + + const { storagePath } = getStoragePaths(repo.dbPath); + const first = await loadMeta(storagePath); + expect(first?.moveIngestAvailable).toBe(false); + if (!first) throw new Error('expected metadata after initial analysis'); + + // Simulate an index whose Move facts were built by a since-vanished + // compiler (e.g. the release cache was wiped between runs). + const identity = { version: '2.0.0', source: 'release', fingerprint: 'fp' } as const; + const compilerBackedMeta: RepoMeta = { + ...first, + moveIngestAvailable: true, + moveCompilerIdentity: identity, + }; + await saveMeta(storagePath, compilerBackedMeta); + + const target = path.join(repo.dbPath, 'src', 'logger.ts'); + await writeFile(target, (await readFile(target, 'utf-8')) + '\n// touched by test\n'); + await expect( + runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }), + ).rejects.toThrow('move-flow is unavailable; the existing Move graph was left unchanged'); + + expect(await loadMeta(storagePath)).toEqual(compilerBackedMeta); + await expect( + runFullAnalysis(repo.dbPath, { skipAgentsMd: true, force: true }, { onProgress: () => {} }), + ).rejects.toThrow('existing Move graph was left unchanged'); + expect(await loadMeta(storagePath)).toEqual(compilerBackedMeta); + } finally { + await repo.cleanup(); + } + }, 300_000); + it('incremental output is byte-equivalent to a full rebuild (incremental ≡ --force on the same repo state)', async () => { // The central correctness contract of this PR: an incremental run // and a full rebuild from the same repo state must produce identical diff --git a/gitnexus/test/unit/move/discovery.test.ts b/gitnexus/test/unit/move/discovery.test.ts new file mode 100644 index 000000000..7c827e503 --- /dev/null +++ b/gitnexus/test/unit/move/discovery.test.ts @@ -0,0 +1,35 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('glob', () => ({ globIterate: vi.fn() })); + +import { globIterate } from 'glob'; +import { repoHasMove } from '../../../src/core/move/discovery.js'; + +const mockGlobIterate = vi.mocked(globIterate); + +const results = async function* (values: string[]): AsyncGenerator { + yield* values; +}; + +describe('repoHasMove', () => { + beforeEach(() => { + mockGlobIterate.mockReset(); + }); + + it('distinguishes an empty repository from a discovery failure', async () => { + mockGlobIterate.mockReturnValueOnce(results([])).mockReturnValueOnce( + (async function* () { + throw new Error('permission denied'); + })(), + ); + + await expect(repoHasMove('/repo')).resolves.toBe(false); + await expect(repoHasMove('/repo')).rejects.toThrow('permission denied'); + }); + + it('reports Move when a manifest is found', async () => { + mockGlobIterate.mockReturnValue(results(['contracts/Move.toml'])); + + await expect(repoHasMove('/repo')).resolves.toBe(true); + }); +}); diff --git a/gitnexus/test/unit/move/incremental-safety.test.ts b/gitnexus/test/unit/move/incremental-safety.test.ts index 38c66c27c..f224c35bb 100644 --- a/gitnexus/test/unit/move/incremental-safety.test.ts +++ b/gitnexus/test/unit/move/incremental-safety.test.ts @@ -1,9 +1,30 @@ import { describe, expect, it } from 'vitest'; import { isMoveCompilerInputPath, - moveAvailabilityRequiresFullRebuild, + moveCompilerRequiresFullRebuild, + persistedMoveGraphRequiresCompiler, + shouldProvisionMoveFlowBeforeFastPath, + type MoveCompilerIdentity, + type MoveFlowReleaseSelection, } from '../../../src/core/move/constants.js'; +const compiler = ( + version = '2.0.0', + source: MoveCompilerIdentity['source'] = 'release', + fingerprint = 'sha-2.0.0', +): MoveCompilerIdentity => ({ version, source, fingerprint }); + +const release = ( + version = '2.0.0', + repository = 'aptos-labs/aptos-ai', + tag = `move-flow-v${version}`, +): MoveFlowReleaseSelection => ({ + version, + repository, + tag, + assetName: `${tag}-aarch64-apple-darwin.zip`, +}); + describe('Move incremental safety', () => { it.each([ 'sources/coin.move', @@ -21,11 +42,59 @@ describe('Move incremental safety', () => { }, ); - it('rebuilds only when a Move repo gains compiler availability', () => { - expect(moveAvailabilityRequiresFullRebuild(true, true, undefined)).toBe(true); - expect(moveAvailabilityRequiresFullRebuild(true, true, false)).toBe(true); - expect(moveAvailabilityRequiresFullRebuild(true, true, true)).toBe(false); - expect(moveAvailabilityRequiresFullRebuild(true, false, true)).toBe(false); - expect(moveAvailabilityRequiresFullRebuild(false, true, undefined)).toBe(false); + it('rebuilds when compiler-backed facts are missing or their producer changes', () => { + expect(moveCompilerRequiresFullRebuild(true, compiler(), undefined, undefined)).toBe(true); + expect(moveCompilerRequiresFullRebuild(true, compiler(), false, undefined)).toBe(true); + expect(moveCompilerRequiresFullRebuild(true, compiler(), true, undefined)).toBe(true); + expect(moveCompilerRequiresFullRebuild(true, compiler(), true, compiler())).toBe(false); + expect(moveCompilerRequiresFullRebuild(true, compiler('2.1.0'), true, compiler())).toBe(true); + expect( + moveCompilerRequiresFullRebuild(true, compiler('2.0.0', 'explicit'), true, compiler()), + ).toBe(true); + expect( + moveCompilerRequiresFullRebuild( + true, + compiler('2.0.0', 'release', 'new-binary'), + true, + compiler(), + ), + ).toBe(true); + expect(moveCompilerRequiresFullRebuild(true, undefined, true, compiler())).toBe(false); + expect(moveCompilerRequiresFullRebuild(false, compiler(), undefined, undefined)).toBe(false); + }); + + it('recognizes compiler-backed legacy metadata from recorded Move inputs', () => { + expect(persistedMoveGraphRequiresCompiler(true, undefined)).toBe(true); + expect(persistedMoveGraphRequiresCompiler(false, { 'Move.toml': 'hash' })).toBe(false); + expect(persistedMoveGraphRequiresCompiler(undefined, { 'Move.toml': 'hash' })).toBe(true); + expect(persistedMoveGraphRequiresCompiler(undefined, { 'src/main.ts': 'hash' })).toBe(false); + }); + + it('provisions before the fast path when a managed release selection changes', () => { + const current = release(); + expect(shouldProvisionMoveFlowBeforeFastPath(true, true, compiler(), current, current)).toBe( + false, + ); + expect( + shouldProvisionMoveFlowBeforeFastPath(true, true, compiler(), current, release('2.1.0')), + ).toBe(true); + expect( + shouldProvisionMoveFlowBeforeFastPath( + true, + true, + compiler(), + current, + release('2.0.0', 'fork/move-flow', 'custom-v2'), + ), + ).toBe(true); + expect( + shouldProvisionMoveFlowBeforeFastPath( + true, + true, + compiler('2.0.0', 'explicit'), + undefined, + current, + ), + ).toBe(false); }); }); diff --git a/gitnexus/test/unit/move/install-move-flow.test.ts b/gitnexus/test/unit/move/install-move-flow.test.ts index d0fa11983..bfc2098c7 100644 --- a/gitnexus/test/unit/move/install-move-flow.test.ts +++ b/gitnexus/test/unit/move/install-move-flow.test.ts @@ -1,5 +1,15 @@ import { EventEmitter } from 'node:events'; -import { existsSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + utimesSync, + writeFileSync, +} from 'node:fs'; import type { FileHandle } from 'node:fs/promises'; import type { ClientRequest, IncomingMessage } from 'node:http'; import { tmpdir } from 'node:os'; @@ -7,17 +17,21 @@ import path from 'node:path'; import { PassThrough } from 'node:stream'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { - acquireInstallLock, downloadToFile, expectedSha, - getMoveFlowInstallConfig, - installMoveFlow, - moveFlowInstallLockWaitMs, - powershellExpandArchiveInvocation, - reportedMoveFlowVersion, - settleMoveFlowCleanup, sha256File, verifyArchiveChecksum, +} from '../../../src/core/move/artifact-download.js'; +import { + acquireInstallLock, + moveFlowInstallLockWaitMs, +} from '../../../src/core/move/install-lock.js'; +import { + findCachedMoveFlowBinary, + getMoveFlowInstallConfig, + installMoveFlow, + powershellExpandArchiveInvocation, + reportedMoveFlowVersion, } from '../../../src/core/move/install.js'; const tempRoots: string[] = []; @@ -62,6 +76,13 @@ describe('move-flow installer', () => { ).resolves.toEqual({ status: 'mismatch', expected, actual }); }); + it('rejects a checksum manifest that omits the selected asset', async () => { + const archive = writeArchive(); + await expect( + verifyArchiveChecksum(`${'0'.repeat(64)} another-asset.zip`, assetName, archive), + ).resolves.toEqual({ status: 'missing' }); + }); + it('rejects a mid-download response error and removes the partial file', async () => { const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-')); tempRoots.push(root); @@ -90,6 +111,233 @@ describe('move-flow installer', () => { expect(existsSync(`${destination}.partial`)).toBe(false); }); + it('settles the response pipeline before cleaning up a request error after headers', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-request-race-')); + tempRoots.push(root); + const destination = path.join(root, 'move-flow.zip'); + let response!: PassThrough & IncomingMessage; + const get = vi.fn((_url, _options, onResponse) => { + const request = new EventEmitter() as ClientRequest; + request.destroy = vi.fn(); + queueMicrotask(() => { + response = new PassThrough() as PassThrough & IncomingMessage; + response.statusCode = 200; + response.headers = {}; + onResponse(response); + response.write('partial archive'); + request.emit('error', new Error('request failed after headers')); + request.emit('close'); + }); + return request; + }); + + await expect( + downloadToFile('https://example.test/move-flow.zip', destination, 1_000, get), + ).rejects.toThrow('request failed after headers'); + expect(response.destroyed).toBe(true); + expect(get).toHaveBeenCalledOnce(); + expect(existsSync(destination)).toBe(false); + expect(existsSync(`${destination}.partial`)).toBe(false); + }); + + it.each([ + ['declared', { 'content-length': '17' }], + ['streamed', {}], + ])('rejects %s downloads that exceed the byte limit', async (kind, headers) => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-limit-')); + tempRoots.push(root); + const destination = path.join(root, 'move-flow.zip'); + let response!: PassThrough & IncomingMessage; + const get = vi.fn((_url, _options, onResponse) => { + const request = new EventEmitter() as ClientRequest; + request.destroy = vi.fn(); + queueMicrotask(() => { + response = new PassThrough() as PassThrough & IncomingMessage; + response.statusCode = 200; + response.headers = headers; + onResponse(response); + response.end('sixteen-byte-body'); + }); + return request; + }); + + await expect( + downloadToFile('https://example.test/move-flow.zip', destination, 1_000, get, 8), + ).rejects.toThrow('download exceeds 8 bytes'); + if (kind === 'declared') expect(response.destroyed).toBe(true); + expect(existsSync(destination)).toBe(false); + expect(existsSync(`${destination}.partial`)).toBe(false); + }); + + it('retries transient HTTP failures within the same deadline', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-retry-')); + tempRoots.push(root); + const destination = path.join(root, 'move-flow.zip'); + let attempt = 0; + const get = vi.fn((_url, _options, onResponse) => { + const request = new EventEmitter() as ClientRequest; + request.destroy = vi.fn((error?: Error) => { + if (error) request.emit('error', error); + return request; + }); + queueMicrotask(() => { + const response = new PassThrough() as IncomingMessage; + response.statusCode = attempt++ === 0 ? 503 : 200; + response.headers = {}; + onResponse(response); + response.end(response.statusCode === 200 ? 'verified archive' : undefined); + request.emit('close'); + }); + return request; + }); + + await expect( + downloadToFile('https://example.test/move-flow.zip', destination, 1_000, get), + ).resolves.toBeUndefined(); + expect(get).toHaveBeenCalledTimes(2); + expect(readFileSync(destination, 'utf8')).toBe('verified archive'); + }); + + it('rejects redirects that downgrade HTTPS', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-redirect-')); + tempRoots.push(root); + const destination = path.join(root, 'move-flow.zip'); + const get = vi.fn((_url, _options, onResponse) => { + const request = new EventEmitter() as ClientRequest; + request.destroy = vi.fn(); + queueMicrotask(() => { + const response = new PassThrough() as IncomingMessage; + response.statusCode = 302; + response.headers = { location: 'http://example.test/insecure.zip' }; + onResponse(response); + request.emit('close'); + }); + return request; + }); + + await expect( + downloadToFile('https://example.test/move-flow.zip', destination, 1_000, get), + ).rejects.toThrow('refusing non-HTTPS redirect'); + expect(get).toHaveBeenCalledOnce(); + }); + + it('rejects an initial non-HTTPS artifact URL before opening a request', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-download-http-')); + tempRoots.push(root); + const get = vi.fn(); + + await expect( + downloadToFile( + 'http://example.test/move-flow.zip', + path.join(root, 'move-flow.zip'), + 1_000, + get, + ), + ).rejects.toThrow('downloads require HTTPS'); + expect(get).not.toHaveBeenCalled(); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects a hash-valid cache whose execute bit was lost', + async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-non-executable-')); + tempRoots.push(root); + const env = { + ...process.env, + GITNEXUS_HOME: root, + GITNEXUS_SKIP_MOVE_FLOW: '0', + }; + const config = await getMoveFlowInstallConfig(env); + expect(config).not.toBeNull(); + if (!config) throw new Error('expected a supported platform'); + mkdirSync(config.installDir, { recursive: true }); + writeFileSync(config.binaryPath, 'cached move-flow'); + chmodSync(config.binaryPath, 0o644); + writeFileSync( + config.metadataPath, + JSON.stringify({ + version: config.version, + repository: config.repository, + tag: config.tag, + assetName: config.assetName, + binarySha256: await sha256File(config.binaryPath), + }), + ); + + await expect(findCachedMoveFlowBinary(env)).resolves.toBeNull(); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'returns the verified descriptor for a live verified cache', + async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-valid-cache-')); + tempRoots.push(root); + const env = { ...process.env, GITNEXUS_HOME: root, GITNEXUS_SKIP_MOVE_FLOW: '0' }; + const config = await getMoveFlowInstallConfig(env); + if (!config) throw new Error('expected a supported platform'); + mkdirSync(config.installDir, { recursive: true }); + writeFileSync(config.binaryPath, '#!/bin/sh\nprintf "move-flow 2.0.0\\n"\n'); + chmodSync(config.binaryPath, 0o755); + writeFileSync( + config.metadataPath, + JSON.stringify({ + schemaVersion: 1, + version: config.version, + repository: config.repository, + tag: config.tag, + assetName: config.assetName, + archiveSha256: '0'.repeat(64), + binarySha256: await sha256File(config.binaryPath), + }), + ); + + await expect(findCachedMoveFlowBinary(env)).resolves.toMatchObject({ + binaryPath: config.binaryPath, + version: '2.0.0', + }); + }, + ); + + it.skipIf(process.platform === 'win32')('rejects a symlinked cached binary', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-symlink-cache-')); + tempRoots.push(root); + const env = { ...process.env, GITNEXUS_HOME: root, GITNEXUS_SKIP_MOVE_FLOW: '0' }; + const config = await getMoveFlowInstallConfig(env); + if (!config) throw new Error('expected a supported platform'); + mkdirSync(config.installDir, { recursive: true }); + const target = path.join(root, 'move-flow-target'); + writeFileSync(target, '#!/bin/sh\nprintf "move-flow 2.0.0\\n"\n'); + chmodSync(target, 0o755); + symlinkSync(target, config.binaryPath); + writeFileSync( + config.metadataPath, + JSON.stringify({ + schemaVersion: 1, + version: config.version, + repository: config.repository, + tag: config.tag, + assetName: config.assetName, + archiveSha256: '0'.repeat(64), + binarySha256: await sha256File(target), + }), + ); + + await expect(findCachedMoveFlowBinary(env)).resolves.toBeNull(); + }); + + it('rejects oversized cache metadata before parsing it', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-move-flow-large-metadata-')); + tempRoots.push(root); + const env = { ...process.env, GITNEXUS_HOME: root, GITNEXUS_SKIP_MOVE_FLOW: '0' }; + const config = await getMoveFlowInstallConfig(env); + if (!config) throw new Error('expected a supported platform'); + mkdirSync(config.installDir, { recursive: true }); + writeFileSync(config.metadataPath, 'x'.repeat(65 * 1024)); + + await expect(findCachedMoveFlowBinary(env)).resolves.toBeNull(); + }); + it('passes PowerShell paths as environment data instead of command source', () => { const archive = `C:\\temp\\move flow's "archive".zip`; const destination = `C:\\temp\\destination's folder`; @@ -123,6 +371,27 @@ describe('move-flow installer', () => { if (process.platform === 'linux') expect(config?.releaseTarget).toContain('compat'); }); + it('keeps compatible release coordinates dynamic', async () => { + const config = await getMoveFlowInstallConfig({ + ...process.env, + GITNEXUS_MOVE_FLOW_VERSION: '2.7.9', + GITNEXUS_MOVE_FLOW_REPO: 'example/move-flow', + GITNEXUS_MOVE_FLOW_TAG: 'release-2.7.9', + }); + + expect(config).toMatchObject({ + version: '2.7.9', + repository: 'example/move-flow', + tag: 'release-2.7.9', + }); + }); + + it('rejects an incompatible configured major before downloading', async () => { + await expect( + getMoveFlowInstallConfig({ ...process.env, GITNEXUS_MOVE_FLOW_VERSION: '3.0.0' }), + ).rejects.toThrow('unsupported move-flow major version'); + }); + it('keeps install waiters alive beyond the full download budget', () => { expect(moveFlowInstallLockWaitMs(30_000)).toBe(120_000); expect(moveFlowInstallLockWaitMs(90_000)).toBe(240_000); @@ -242,14 +511,4 @@ describe('move-flow installer', () => { await release(); expect(existsSync(lockPath)).toBe(false); }); - - it('runs lock release even when another cleanup task rejects', async () => { - const release = vi.fn(async () => {}); - await expect( - settleMoveFlowCleanup(async () => { - throw new Error('simulated temp cleanup failure'); - }, release), - ).resolves.toBeUndefined(); - expect(release).toHaveBeenCalledOnce(); - }); }); diff --git a/gitnexus/test/unit/move/mcp-client.test.ts b/gitnexus/test/unit/move/mcp-client.test.ts index 27742965d..e3d4a35e0 100644 --- a/gitnexus/test/unit/move/mcp-client.test.ts +++ b/gitnexus/test/unit/move/mcp-client.test.ts @@ -11,7 +11,7 @@ vi.mock('node:child_process', () => ({ import { MoveFlowMcpClient, MoveFlowToolCallError, - tryCreateMoveFlowClient, + tryResolveMoveFlowClient, detectMoveFlowCapabilities, } from '../../../src/core/move/mcp-client.js'; import { spawn, execFileSync } from 'node:child_process'; @@ -29,16 +29,15 @@ function createMockProc() { return proc; } -describe('tryCreateMoveFlowClient', () => { +describe('tryResolveMoveFlowClient', () => { beforeEach(() => { vi.resetAllMocks(); delete process.env.MOVE_FLOW; }); - it('returns MoveFlowMcpClient when binary is found', () => { + it('resolves a client when the binary is found', () => { mockExecFileSync.mockReturnValue('move-flow 2.0.0'); - const client = tryCreateMoveFlowClient(); - expect(client).toBeInstanceOf(MoveFlowMcpClient); + expect(tryResolveMoveFlowClient()?.client).toBeInstanceOf(MoveFlowMcpClient); expect(mockExecFileSync).toHaveBeenCalledWith( 'move-flow', ['--version'], @@ -46,11 +45,21 @@ describe('tryCreateMoveFlowClient', () => { ); }); + it('returns the reported version with the resolved client', () => { + mockExecFileSync.mockReturnValue('move-flow 2.3.4-rc1'); + + expect(tryResolveMoveFlowClient('/custom/move-flow')).toMatchObject({ + client: expect.any(MoveFlowMcpClient), + version: '2.3.4-rc1', + }); + expect(mockExecFileSync).toHaveBeenCalledOnce(); + }); + it('returns null when binary is not found', () => { mockExecFileSync.mockImplementation(() => { throw new Error('not found'); }); - expect(tryCreateMoveFlowClient()).toBeNull(); + expect(tryResolveMoveFlowClient()).toBeNull(); }); it.each([ @@ -60,8 +69,7 @@ describe('tryCreateMoveFlowClient', () => { ])('passes an explicit binary path directly to execFileSync: %s', (binary) => { process.env.MOVE_FLOW = binary; mockExecFileSync.mockReturnValue('move-flow 2.0.0'); - const client = tryCreateMoveFlowClient(); - expect(client).toBeInstanceOf(MoveFlowMcpClient); + expect(tryResolveMoveFlowClient()?.client).toBeInstanceOf(MoveFlowMcpClient); expect(mockExecFileSync).toHaveBeenCalledWith(binary, ['--version'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], @@ -73,7 +81,7 @@ describe('tryCreateMoveFlowClient', () => { 'rejects an incompatible version response: %s', (versionOutput) => { mockExecFileSync.mockReturnValue(versionOutput); - expect(tryCreateMoveFlowClient('/custom/move-flow')).toBeNull(); + expect(tryResolveMoveFlowClient('/custom/move-flow')).toBeNull(); }, ); @@ -81,7 +89,9 @@ describe('tryCreateMoveFlowClient', () => { 'accepts a compatible-major version response: %s', (versionOutput) => { mockExecFileSync.mockReturnValue(versionOutput); - expect(tryCreateMoveFlowClient('/custom/move-flow')).toBeInstanceOf(MoveFlowMcpClient); + expect(tryResolveMoveFlowClient('/custom/move-flow')?.client).toBeInstanceOf( + MoveFlowMcpClient, + ); }, ); }); @@ -196,6 +206,77 @@ describe('MoveFlowMcpClient', () => { await vi.advanceTimersByTimeAsync(25); await failure; expect(proc.kill).toHaveBeenCalledOnce(); + expect((client as any).proc).toBeNull(); + expect((client as any).initialized).toBe(false); + expect((client as any).initPromise).toBeNull(); + expect((client as any).pending.size).toBe(0); + await client.shutdown(); + }); + + it('ignores a late response after a tool timeout and starts a fresh child', async () => { + vi.useFakeTimers(); + process.env.GITNEXUS_MOVE_FLOW_TIMEOUT_MS = '25'; + const timedOutProc = createMockProc(); + const retryProc = createMockProc(); + mockSpawn.mockReturnValueOnce(timedOutProc as any).mockReturnValueOnce(retryProc as any); + + timedOutProc.stdin.on('data', (chunk: Buffer) => { + for (const line of chunk.toString().split('\n')) { + if (!line.trim()) continue; + const msg = JSON.parse(line); + if (msg.method === 'initialize') { + timedOutProc.stdout.write( + JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: {} }) + '\n', + ); + } + } + }); + const client = new MoveFlowMcpClient('move-flow'); + const first = client.facts('/slow'); + const failure = expect(first).rejects.toThrow("move-flow 'move_package_query' timed out"); + await vi.advanceTimersByTimeAsync(25); + await failure; + + timedOutProc.stdout.write( + JSON.stringify({ + jsonrpc: '2.0', + id: 2, + result: { content: [{ type: 'text', text: '{"late":true}' }] }, + }) + '\n', + ); + expect((client as any).pending.size).toBe(0); + + serveToolsCall(retryProc, { + result: { content: [{ type: 'text', text: '{"fresh":true}' }], isError: false }, + }); + await expect(client.facts('/retry')).resolves.toEqual({ fresh: true }); + expect(mockSpawn).toHaveBeenCalledTimes(2); + await client.shutdown(); + }); + + it('propagates a capabilities transport timeout and clears it for retry', async () => { + vi.useFakeTimers(); + const proc = createMockProc(); + mockSpawn.mockReturnValue(proc as any); + proc.stdin.on('data', (chunk: Buffer) => { + for (const line of chunk.toString().split('\n')) { + if (!line.trim()) continue; + const msg = JSON.parse(line); + if (msg.method === 'initialize') { + proc.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: {} }) + '\n'); + } + } + }); + + const client = new MoveFlowMcpClient('move-flow'); + const capabilities = client.capabilities(); + const failure = expect(capabilities).rejects.toThrow("move-flow 'tools/list' timed out"); + await vi.advanceTimersByTimeAsync(30_000); + await failure; + + expect(proc.kill).toHaveBeenCalledOnce(); + expect((client as any).capsPromise).toBeNull(); + expect((client as any).proc).toBeNull(); await client.shutdown(); }); diff --git a/gitnexus/test/unit/move/provision.test.ts b/gitnexus/test/unit/move/provision.test.ts index 13050ad28..e56009275 100644 --- a/gitnexus/test/unit/move/provision.test.ts +++ b/gitnexus/test/unit/move/provision.test.ts @@ -1,9 +1,13 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { MoveFlowMcpClient } from '../../../src/core/move/mcp-client.js'; +import type { + MoveFlowMcpClient, + ResolvedMoveFlowClient, +} from '../../../src/core/move/mcp-client.js'; import { - ensureMoveFlowClient, + ensureMoveFlowRuntime, type MoveFlowProvisionDependencies, } from '../../../src/core/move/provision.js'; +import type { VerifiedMoveFlowBinary } from '../../../src/core/move/install.js'; const originalMoveFlow = process.env.MOVE_FLOW; @@ -13,95 +17,146 @@ afterEach(() => { }); const client = {} as MoveFlowMcpClient; +const resolved: ResolvedMoveFlowClient = { client, version: '2.0.0' }; +const cached: VerifiedMoveFlowBinary = { + binaryPath: '/cache/move-flow', + version: '2.0.0', + fingerprint: 'release-fingerprint', +}; -describe('ensureMoveFlowClient', () => { +const dependencies = ( + overrides: Partial = {}, +): MoveFlowProvisionDependencies => ({ + resolveClient: vi.fn(() => null), + createClient: vi.fn(() => client), + findCached: vi.fn(async () => null), + install: vi.fn(async () => ({ status: 'failed', message: 'offline' })), + ...overrides, +}); + +describe('ensureMoveFlowRuntime', () => { it('does not install when an explicit or PATH binary already resolves', async () => { - const dependencies: MoveFlowProvisionDependencies = { - resolveClient: vi.fn(() => client), - findCached: vi.fn(async () => null), - install: vi.fn(), - }; + const deps = dependencies({ resolveClient: vi.fn(() => resolved) }); - await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBe(client); - expect(dependencies.install).not.toHaveBeenCalled(); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); + expect(deps.install).not.toHaveBeenCalled(); }); it('keeps an invalid explicit MOVE_FLOW authoritative', async () => { process.env.MOVE_FLOW = '/missing/move-flow'; - const dependencies: MoveFlowProvisionDependencies = { - resolveClient: vi.fn(() => null), - findCached: vi.fn(async () => null), - install: vi.fn(), - }; + const deps = dependencies(); - await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); - expect(dependencies.install).not.toHaveBeenCalled(); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + expect(deps.install).not.toHaveBeenCalled(); expect(process.env.MOVE_FLOW).toBe('/missing/move-flow'); }); - it('installs once, publishes the cache path to the resolver, and returns a client', async () => { + it('supports local-only resolution without starting installation', async () => { delete process.env.MOVE_FLOW; - const resolveClient = vi - .fn<(binaryPath?: string) => MoveFlowMcpClient | null>() - .mockReturnValueOnce(null) - .mockReturnValueOnce(client); - const install = vi.fn(async () => ({ - status: 'installed' as const, - binaryPath: '/cache/move-flow', - })); + const deps = dependencies(); - await expect( - ensureMoveFlowClient({}, { resolveClient, findCached: async () => null, install }), - ).resolves.toBe(client); - expect(install).toHaveBeenCalledOnce(); - expect(resolveClient).toHaveBeenLastCalledWith('/cache/move-flow'); - expect(process.env.MOVE_FLOW).toBeUndefined(); + await expect(ensureMoveFlowRuntime({ install: false }, deps)).resolves.toBeNull(); + expect(deps.install).not.toHaveBeenCalled(); }); - it('soft-fails when installation rejects', async () => { + it('installs once and returns the verified release identity', async () => { delete process.env.MOVE_FLOW; - const onLog = vi.fn(); - const dependencies: MoveFlowProvisionDependencies = { - resolveClient: vi.fn(() => null), - findCached: vi.fn(async () => null), - install: vi.fn(async () => { - throw new Error('offline'); - }), - }; + const deps = dependencies({ + install: vi.fn(async () => ({ status: 'installed' as const, binary: cached })), + }); - await expect(ensureMoveFlowClient({ onLog }, dependencies)).resolves.toBeNull(); - expect(onLog).toHaveBeenCalledWith(expect.stringContaining('offline')); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toEqual({ + client, + identity: { + version: '2.0.0', + source: 'release', + fingerprint: 'release-fingerprint', + }, + }); + expect(deps.install).toHaveBeenCalledOnce(); + expect(deps.createClient).toHaveBeenCalledWith('/cache/move-flow'); + expect(deps.resolveClient).toHaveBeenCalledTimes(1); }); - it('does not retry a failed installation within the same process', async () => { + it('retries after a rejected installation', async () => { delete process.env.MOVE_FLOW; - const dependencies: MoveFlowProvisionDependencies = { - resolveClient: vi.fn(() => null), - findCached: vi.fn(async () => null), - install: vi.fn(async () => { - throw new Error('offline'); - }), - }; + const install = vi + .fn() + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValueOnce({ status: 'installed', binary: cached }); + const deps = dependencies({ install }); - await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); - await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBeNull(); - expect(dependencies.install).toHaveBeenCalledOnce(); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); + expect(install).toHaveBeenCalledTimes(2); }); - it('uses the verified user cache before PATH candidates', async () => { + it('retries after a soft installation failure', async () => { delete process.env.MOVE_FLOW; - const resolveClient = vi.fn((binaryPath?: string) => - binaryPath === '/cache/move-flow' ? client : null, + const install = vi + .fn() + .mockResolvedValueOnce({ status: 'failed', message: 'checksum service unavailable' }) + .mockResolvedValueOnce({ status: 'installed', binary: cached }); + const deps = dependencies({ install }); + + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); + expect(install).toHaveBeenCalledTimes(2); + }); + + it('shares only an in-flight install attempt', async () => { + delete process.env.MOVE_FLOW; + let finish!: (value: { status: 'installed'; binary: VerifiedMoveFlowBinary }) => void; + const pending = new Promise<{ status: 'installed'; binary: VerifiedMoveFlowBinary }>( + (resolve) => { + finish = resolve; + }, ); - const dependencies: MoveFlowProvisionDependencies = { - resolveClient, - findCached: vi.fn(async () => '/cache/move-flow'), - install: vi.fn(), - }; + const install = vi.fn(() => pending); + const deps = dependencies({ install }); - await expect(ensureMoveFlowClient({}, dependencies)).resolves.toBe(client); - expect(resolveClient).toHaveBeenCalledTimes(1); - expect(resolveClient).toHaveBeenCalledWith('/cache/move-flow'); - expect(dependencies.install).not.toHaveBeenCalled(); + const first = ensureMoveFlowRuntime({}, deps); + const second = ensureMoveFlowRuntime({}, deps); + await vi.waitFor(() => expect(install).toHaveBeenCalledOnce()); + finish({ status: 'installed', binary: cached }); + + await expect(Promise.all([first, second])).resolves.toHaveLength(2); + expect(deps.createClient).toHaveBeenCalledTimes(2); + }); + + it('uses the verified cache without another version probe', async () => { + delete process.env.MOVE_FLOW; + const deps = dependencies({ findCached: vi.fn(async () => cached) }); + + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ + client, + identity: { source: 'release', fingerprint: 'release-fingerprint' }, + }); + expect(deps.resolveClient).not.toHaveBeenCalled(); + expect(deps.createClient).toHaveBeenCalledWith('/cache/move-flow'); + expect(deps.install).not.toHaveBeenCalled(); + }); + + it('rejects an incompatible verified cache without constructing a client', async () => { + delete process.env.MOVE_FLOW; + const deps = dependencies({ + findCached: vi.fn(async () => ({ ...cached, version: '3.0.0' })), + }); + + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + expect(deps.createClient).not.toHaveBeenCalled(); + }); + + it('rejects an incompatible installed result without constructing a client', async () => { + delete process.env.MOVE_FLOW; + const deps = dependencies({ + install: vi.fn(async () => ({ + status: 'installed' as const, + binary: { ...cached, version: '3.0.0' }, + })), + }); + + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + expect(deps.createClient).not.toHaveBeenCalled(); }); }); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 3a7890544..0a115a24c 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -91,6 +91,7 @@ export default defineConfig({ 'test/integration/lbug-delete-nodes-for-files.test.ts', 'test/integration/lbug-query-importers-batch.test.ts', 'test/integration/move-mixed-language-roundtrip.test.ts', + 'test/integration/move-run-analyze-e2e.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', 'test/unit/incremental-orchestration.test.ts', ], @@ -140,6 +141,7 @@ export default defineConfig({ 'test/integration/lbug-delete-nodes-for-files.test.ts', 'test/integration/lbug-query-importers-batch.test.ts', 'test/integration/move-mixed-language-roundtrip.test.ts', + 'test/integration/move-run-analyze-e2e.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', 'test/unit/incremental-orchestration.test.ts', ], From cec1a74cf0efc16de0f87c76d5efafb299a88515 Mon Sep 17 00:00:00 2001 From: zwxxb Date: Tue, 21 Jul 2026 23:58:33 +0200 Subject: [PATCH 3/4] fix(move): harden provisioning failure handling Avoid repeated install attempts and prove compiler-backed indexes remain intact when move-flow disappears. Align docs and CI metadata with managed runtime behavior. Co-authored-by: Cursor --- .github/workflows/ci-tests.yml | 6 +- README.md | 29 +++- gitnexus/README.md | 20 ++- gitnexus/src/core/lbug/csv-generator.ts | 4 + gitnexus/src/core/move/README.md | 27 ++++ gitnexus/src/core/move/artifact-download.ts | 6 +- gitnexus/src/core/move/install-lock.ts | 6 +- gitnexus/src/core/move/install.ts | 22 +-- gitnexus/src/core/move/provision.ts | 12 +- .../integration/move-run-analyze-e2e.test.ts | 149 +++++++++++++----- .../unit/incremental-orchestration.test.ts | 46 ------ gitnexus/test/unit/move/provision.test.ts | 28 ++-- gitnexus/test/unit/node-table-layout.test.ts | 20 +++ 13 files changed, 247 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 87a7f229a..a6d7b58cc 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -44,7 +44,7 @@ jobs: # (metadata + binary sha256 + version probe) before use. Keep the key # tied to MOVE_FLOW_RELEASE.version in gitnexus/src/core/move/release.ts. - name: Cache move-flow binary - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.gitnexus/tools/move-flow key: ${{ runner.os }}-move-flow-v2.0.0 @@ -58,7 +58,7 @@ jobs: # resolves the extension at module load and can't self-install, so sharding # could otherwise drop it into a shard with no installer sibling. - name: Cache LadybugDB FTS extension - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.lbdb/extension key: lbug-fts-${{ runner.os }}-${{ hashFiles('gitnexus/package-lock.json') }} @@ -246,7 +246,7 @@ jobs: # never a correctness dependency. Keyed by lockfile hash so a LadybugDB # version bump re-installs; per-OS because the extension is a native binary. - name: Cache LadybugDB FTS extension - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.lbdb/extension key: lbug-fts-${{ runner.os }}-${{ hashFiles('gitnexus/package-lock.json') }} diff --git a/README.md b/README.md index cb4ac5fac..fafb1388f 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ That's it. `analyze` indexes the codebase, installs agent skills, registers Clau > **Fastest MCP startup:** install globally (`npm i -g gitnexus`) before running `gitnexus setup` — this writes an absolute-path MCP config that bypasses `npx` entirely. On a cold cache, an `npx`-based MCP install can exceed Claude Code's `MCP_TIMEOUT` default (~30s). -> **No C++ toolchain?** Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four languages won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. +> **No C++ toolchain?** Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four languages won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. At analyze time, strict `=1` also disables automatic `move-flow` downloads; an explicit `MOVE_FLOW`, verified cache entry, or compatible binary on `PATH` can still provide Move support. > **Behind an HTTP proxy / regional firewall?** `onnxruntime-node`'s postinstall downloads optional CUDA binaries from `api.nuget.org` and ignores `HTTP_PROXY`/`HTTPS_PROXY` ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). The embedding stack is an optional dependency, so a failed download no longer breaks the install — and it self-heals: the first `gitnexus analyze --embeddings` (or `gitnexus embeddings install`) fetches the stack through your npm registry config (mirrors/proxies apply, no NuGet) into `~/.gitnexus/embedding-runtime` (override with `GITNEXUS_EMBEDDING_RUNTIME_DIR`). The on-demand prefix needs Node with `module.registerHooks` (≥ 22.15 on 22.x, ≥ 23.5 on 23.x); on older Node, keep the stack in the install itself with `ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus` (works on every supported Node). @@ -90,7 +90,7 @@ That's it. `analyze` indexes the codebase, installs agent skills, registers Clau | **Install** | `npm install -g gitnexus` | No install — [gitnexus.vercel.app](https://gitnexus.vercel.app) | | **Storage** | LadybugDB native (fast, persistent) | LadybugDB WASM (in-memory, per session) | | **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Privacy** | Everything local, no network | Everything in-browser, no server | +| **Privacy** | Source stays local; optional runtimes may be downloaded | Everything in-browser, no server | > **Bridge mode:** `gitnexus serve` connects the two — the web UI auto-detects the local server and can browse all your CLI-indexed repos without re-uploading or re-indexing. @@ -466,6 +466,29 @@ Notes: +
+Move compiler runtime + +Repositories containing `Move.toml` use the compiler-backed `move-flow` runtime. `gitnexus analyze` resolves it in this order: + +1. the explicit `MOVE_FLOW` path; +2. a checksum- and version-verified cache under `~/.gitnexus/tools/move-flow`; +3. a compatible `move-flow` on `PATH`; +4. the pinned GitHub release, downloaded over HTTPS and verified against its `SHA256SUMS`. + +The managed install is serialized across processes and published atomically. For air-gapped hosts, install a compatible binary ahead of time and set `MOVE_FLOW`, or pre-seed the managed cache; set `GITNEXUS_SKIP_MOVE_FLOW=1` to disable automatic downloads. An invalid explicit `MOVE_FLOW` remains authoritative and does not fall back to a network install. + +GitNexus records the compiler identity that produced Move facts. A compiler change forces a full rebuild; if a previously compiler-backed graph needs updating while `move-flow` is unavailable, analysis fails before mutating the existing graph. Managed identities include the verified binary hash; after replacing an explicit or `PATH` binary with a different same-version build, run `gitnexus analyze --force`. Rolling back this behavior only requires reverting the GitNexus change; removing the managed cache is optional, and reanalysis is needed only when restoring desired compiler-derived facts. + +Move runtime controls: + +- `MOVE_FLOW` selects an authoritative executable; `GITNEXUS_SKIP_MOVE_FLOW=1` disables automatic downloads. +- `GITNEXUS_MOVE_FLOW_DIR` changes the cache root; `GITNEXUS_MOVE_FLOW_HTTP_TIMEOUT_MS` changes the 30-second per-download deadline. +- `GITNEXUS_MOVE_FLOW_TIMEOUT_MS` changes the five-minute compiler-tool timeout; `GITNEXUS_MOVE_FLOW_COMPAT=1` forces the Linux compatibility build. +- `GITNEXUS_MOVE_FLOW_VERSION`, `GITNEXUS_MOVE_FLOW_REPO`, and `GITNEXUS_MOVE_FLOW_TAG` override trusted release coordinates for advanced testing. + +
+
Environment variables @@ -492,7 +515,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction. On breach the file keeps the captures accumulated so far and logs a warning — the worker returns to JS instead of stalling in native-heavy loops (#2432). `0` expires immediately. | Pathological generated C++ that still exceeds the budget after the indexed lookups; raise for completeness, lower to fail-fast. | | `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | | `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | -| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips optional grammar materialization at install time, disables those grammars at runtime, and disables automatic `move-flow` downloads. Existing verified/cache/PATH Move runtimes remain usable. | Running without optional native runtimes and accepting that affected languages may not be indexed. | | `GITNEXUS_MCP_READ_ONLY` | unset | Set to `1` to expose only proven single-repository read tools and resources; `0` disables the policy and any other value fails startup. | The MCP server runs in an environment where graph mutation, raw Cypher, and cross-repository group routing must be unavailable. | | `GITNEXUS_MCP_ALLOWED_REPOS` | unset | Comma-separated allowlist of canonical indexed repository names or absolute paths. Invalid, ambiguous, or blank entries fail startup. | One MCP process must expose only a bounded subset of the repositories in the global registry. | | `GITNEXUS_MCP_DEFAULT_REPO` | unset | Canonical indexed repository name or absolute path used when a tool or resource omits its repository. Must belong to the allowlist when one is set. | Several repositories are available but unqualified MCP calls should resolve deterministically. | diff --git a/gitnexus/README.md b/gitnexus/README.md index 6c83b27dc..4732d1386 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -423,7 +423,7 @@ gitnexus serve ### Installation fails with native module errors -Some optional language grammars (Dart, Proto, Swift, Kotlin) require native compilation. If they fail, GitNexus still works — those languages will be skipped. To skip them intentionally (no C++ toolchain needed), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before installing. +Some optional language grammars (Dart, Proto, Swift, Kotlin) require native compilation. If they fail, GitNexus still works — those languages will be skipped. To skip them intentionally (no C++ toolchain needed), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before installing. Strict `=1` also disables automatic `move-flow` downloads during analysis; a verified cache entry, explicit `MOVE_FLOW`, or compatible binary on `PATH` remains usable. If `npm install -g gitnexus` fails on native modules: @@ -436,6 +436,24 @@ If `npm install -g gitnexus` fails on native modules: npm install -g gitnexus ``` +### Move compiler provisioning + +When a repository contains `Move.toml`, `gitnexus analyze` resolves `move-flow` +from the authoritative `MOVE_FLOW` path, a verified cache under +`~/.gitnexus/tools/move-flow`, `PATH`, or the pinned GitHub release. Managed +downloads use HTTPS, checksum and version verification, a cross-process lock, +and atomic publication. + +For air-gapped hosts, set `MOVE_FLOW` to a preinstalled compatible binary or +pre-seed the managed cache. Set `GITNEXUS_SKIP_MOVE_FLOW=1` to disable automatic +downloads. The cache root and network/compiler timeouts can be configured with +`GITNEXUS_MOVE_FLOW_DIR`, `GITNEXUS_MOVE_FLOW_HTTP_TIMEOUT_MS`, and +`GITNEXUS_MOVE_FLOW_TIMEOUT_MS`. + +GitNexus records the compiler identity that produced Move facts. A compiler +change forces a full rebuild; if an existing compiler-backed graph needs an +update while `move-flow` is unavailable, analysis fails before mutating it. + ### Installation fails behind an HTTP proxy (`onnxruntime-node` postinstall) `onnxruntime-node`'s postinstall downloads optional CUDA GPU binaries from `api.nuget.org` — outside the npm registry, so registry mirrors don't cover it, and its proxy layer (`global-agent`) ignores the standard `HTTP_PROXY`/`HTTPS_PROXY` variables and rejects 302 redirects ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 540d88673..7e424a4e5 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -275,6 +275,10 @@ export const buildLayoutNodeRow = ( return escapeCSVField(formatFtsDescription(String(value ?? ''))); case 'string': return escapeCSVField(String(value ?? '')); + default: { + const _exhaustive: never = spec.csvEncoding; + throw new Error(`Unsupported CSV column encoding: ${String(_exhaustive)}`); + } } }) .join(','); diff --git a/gitnexus/src/core/move/README.md b/gitnexus/src/core/move/README.md index 6d9bb9c3e..ae82d097e 100644 --- a/gitnexus/src/core/move/README.md +++ b/gitnexus/src/core/move/README.md @@ -10,6 +10,33 @@ Cold compiler builds for large packages may take several minutes. Tool calls default to a five-minute timeout; override it in milliseconds with `GITNEXUS_MOVE_FLOW_TIMEOUT_MS` when a repository needs a larger budget. +## Runtime provisioning + +When `analyze` finds a `Move.toml`, it resolves `move-flow` from the authoritative +`MOVE_FLOW` path, the verified managed cache, `PATH`, or finally the pinned +release in `release.ts`. Managed releases live under +`~/.gitnexus/tools/move-flow` by default, are downloaded over HTTPS, checked +against `SHA256SUMS`, version-probed, and atomically published while a +heartbeat lease serializes concurrent installers. + +Set `GITNEXUS_SKIP_MOVE_FLOW=1` to disable automatic downloads. The umbrella +`GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` does the same while also disabling optional +grammars. Air-gapped hosts should set `MOVE_FLOW` to a preinstalled compatible +binary or pre-seed the managed cache. Advanced trusted-release overrides are +`GITNEXUS_MOVE_FLOW_DIR`, `GITNEXUS_MOVE_FLOW_VERSION`, +`GITNEXUS_MOVE_FLOW_REPO`, `GITNEXUS_MOVE_FLOW_TAG`, +`GITNEXUS_MOVE_FLOW_COMPAT`, and `GITNEXUS_MOVE_FLOW_HTTP_TIMEOUT_MS`. + +The repository metadata records the compiler identity used for Move facts. +Changing that identity forces a full rebuild. If an existing compiler-backed +graph needs updating while the compiler is unavailable, analysis fails before +mutating the graph; restore `move-flow` or set `MOVE_FLOW` and retry. +Managed identities include the verified binary hash; explicit and `PATH` +identities use their locator and reported version, so replace a same-version +local build with `gitnexus analyze --force`. +Filesystem errors while discovering `Move.toml` are also surfaced instead of +silently treating the repository as non-Move. + ```text Move package -> move-flow MCP diff --git a/gitnexus/src/core/move/artifact-download.ts b/gitnexus/src/core/move/artifact-download.ts index a45441050..209e0b512 100644 --- a/gitnexus/src/core/move/artifact-download.ts +++ b/gitnexus/src/core/move/artifact-download.ts @@ -5,6 +5,7 @@ import type { ClientRequest, IncomingMessage } from 'node:http'; import { get as httpsGet, type RequestOptions } from 'node:https'; import { Transform } from 'node:stream'; import { pipeline } from 'node:stream/promises'; +import { setTimeout as wait } from 'node:timers/promises'; export type HttpsGet = ( url: string | URL, @@ -45,11 +46,6 @@ const parseRetryAfter = (value: string | string[] | undefined): number => { return Number.isFinite(date) ? Math.max(0, date - Date.now()) : 0; }; -const wait = (ms: number): Promise => - new Promise((resolve) => { - setTimeout(resolve, ms); - }); - const safeRequestLabel = (target: string): string => { const parsed = new URL(target); return `${parsed.origin}${parsed.pathname}`; diff --git a/gitnexus/src/core/move/install-lock.ts b/gitnexus/src/core/move/install-lock.ts index 91c98ab8f..59d347251 100644 --- a/gitnexus/src/core/move/install-lock.ts +++ b/gitnexus/src/core/move/install-lock.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import type { Stats } from 'node:fs'; import { type FileHandle, mkdir, open, readFile, rm, stat, utimes } from 'node:fs/promises'; import path from 'node:path'; +import { setTimeout as wait } from 'node:timers/promises'; export interface LockOptions { waitTimeoutMs?: number; @@ -28,11 +29,6 @@ const defaultLockIo: InstallLockIo = { }, }; -const wait = (ms: number): Promise => - new Promise((resolve) => { - setTimeout(resolve, ms); - }); - /** Allow for the artifact download, extraction, publication, and version probe. */ export const moveFlowInstallLockWaitMs = (httpTimeoutMs: number): number => Math.max(DEFAULT_LOCK_WAIT_MS, httpTimeoutMs * 2 + INSTALL_COMPLETION_GRACE_MS); diff --git a/gitnexus/src/core/move/install.ts b/gitnexus/src/core/move/install.ts index 679a8e72c..30d10c06f 100644 --- a/gitnexus/src/core/move/install.ts +++ b/gitnexus/src/core/move/install.ts @@ -17,6 +17,7 @@ import { import { get as httpsGet } from 'node:https'; import os from 'node:os'; import path from 'node:path'; +import { promisify } from 'node:util'; import { downloadToFile, MAX_ARCHIVE_BYTES, @@ -81,6 +82,7 @@ const DEFAULT_HTTP_TIMEOUT_MS = 30_000; const MAX_CHECKSUM_BYTES = 1024 * 1024; const MAX_BINARY_BYTES = 128 * 1024 * 1024; const MAX_METADATA_BYTES = 64 * 1024; +const execFileAsync = promisify(execFile); const parsePositiveInteger = (value: string | undefined, fallback: number): number => { const parsed = Number(value); @@ -116,21 +118,11 @@ const execFileOutput = ( args: string[], options: { timeout: number; env?: NodeJS.ProcessEnv; encoding?: BufferEncoding }, ): Promise => - new Promise((resolve, reject) => { - execFile( - file, - args, - { - timeout: options.timeout, - env: options.env, - encoding: options.encoding ?? 'utf8', - }, - (error, stdout) => { - if (error) reject(error); - else resolve(String(stdout)); - }, - ); - }); + execFileAsync(file, args, { + timeout: options.timeout, + env: options.env, + encoding: options.encoding ?? 'utf8', + }).then(({ stdout }) => String(stdout)); let detectedLinuxCompatibility: Promise | undefined; diff --git a/gitnexus/src/core/move/provision.ts b/gitnexus/src/core/move/provision.ts index 2cba7007a..03c63a5e0 100644 --- a/gitnexus/src/core/move/provision.ts +++ b/gitnexus/src/core/move/provision.ts @@ -76,10 +76,14 @@ const getInstallAttempt = ( const created = Promise.resolve().then(dependencies.install); installAttempts.set(dependencies, created); - const clear = (): void => { - if (installAttempts.get(dependencies) === created) installAttempts.delete(dependencies); - }; - void created.then(clear, clear); + void created.then( + (result) => { + if (result.binary && installAttempts.get(dependencies) === created) { + installAttempts.delete(dependencies); + } + }, + () => {}, + ); return created; }; diff --git a/gitnexus/test/integration/move-run-analyze-e2e.test.ts b/gitnexus/test/integration/move-run-analyze-e2e.test.ts index 11265977b..d2710194a 100644 --- a/gitnexus/test/integration/move-run-analyze-e2e.test.ts +++ b/gitnexus/test/integration/move-run-analyze-e2e.test.ts @@ -1,24 +1,102 @@ +import { execFileSync } from 'node:child_process'; import fs from 'node:fs/promises'; import path from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { closeLbug, executeQuery, initLbug } from '../../src/core/lbug/lbug-adapter.js'; +import { getMoveFlowInstallConfig } from '../../src/core/move/install.js'; +import { ensureMoveFlowRuntime } from '../../src/core/move/provision.js'; +import { MOVE_FLOW_RELEASE } from '../../src/core/move/release.js'; import { runFullAnalysis } from '../../src/core/run-analyze.js'; -import { tryResolveMoveFlowClient } from '../../src/core/move/mcp-client.js'; import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; import { createTempDir } from '../helpers/test-db.js'; const requireMoveFlow = process.env.GITNEXUS_REQUIRE_MOVE_FLOW === '1'; const skipMoveFlow = process.env.GITNEXUS_SKIP_MOVE_FLOW === '1'; -const canRun = requireMoveFlow || (!skipMoveFlow && tryResolveMoveFlowClient() !== null); +const runtime = + !requireMoveFlow && skipMoveFlow + ? null + : await ensureMoveFlowRuntime({ + install: requireMoveFlow, + onLog: requireMoveFlow + ? (message) => console.info(`[move-run-analyze] ${message}`) + : undefined, + }); +if (requireMoveFlow && !runtime) { + throw new Error('GITNEXUS_REQUIRE_MOVE_FLOW=1 but MoveFlow provisioning failed'); +} +const expectedCompilerIdentity = runtime?.identity; +await runtime?.client.shutdown(); +const managedConfig = + expectedCompilerIdentity?.source === 'release' ? await getMoveFlowInstallConfig() : null; +const managedCacheRoot = managedConfig + ? path.dirname(path.dirname(managedConfig.installDir)) + : undefined; -describe.skipIf(!canRun)('Move runFullAnalysis persistence', () => { - it('provisions, persists, and queries a real mixed repository', async () => { +const analysisOptions = { + skipAgentsMd: true, + skipSkills: true, + noStats: true, + workerPoolSize: 2, +} as const; + +const analyze = (repoPath: string, force: boolean) => + runFullAnalysis(repoPath, { ...analysisOptions, force }, { onProgress: () => {} }); + +interface GraphSnapshot { + nodes: Array>; + relationships: Array>; +} + +const queryGraphSnapshot = async (): Promise => ({ + nodes: (await executeQuery( + 'MATCH (n) RETURN labels(n) AS label, n AS node ORDER BY n.id', + )) as Array>, + relationships: (await executeQuery( + 'MATCH (source)-[relation:CodeRelation]->(target) ' + + 'RETURN source.id AS sourceId, target.id AS targetId, relation.type AS type, ' + + 'relation.confidence AS confidence, relation.reason AS reason, relation.step AS step ' + + 'ORDER BY sourceId, targetId, type, reason, step', + )) as Array>, +}); + +const readGraphSnapshot = async (lbugPath: string): Promise => { + await initLbug(lbugPath); + try { + return await queryGraphSnapshot(); + } finally { + await closeLbug(); + } +}; + +const initializeGitRepo = (repoPath: string): void => { + execFileSync('git', ['init', '-q'], { cwd: repoPath }); + execFileSync('git', ['add', '.'], { cwd: repoPath }); + execFileSync( + 'git', + [ + '-c', + 'user.name=GitNexus Test', + '-c', + 'user.email=test@gitnexus.local', + '-c', + 'commit.gpgsign=false', + 'commit', + '-q', + '-m', + 'initial', + ], + { cwd: repoPath }, + ); +}; + +describe.skipIf(!expectedCompilerIdentity)('Move runFullAnalysis persistence', () => { + it('preserves a real compiler-backed graph when the compiler disappears', async () => { const repo = await createTempDir('gitnexus-move-run-analyze-'); const home = await createTempDir('gitnexus-move-run-analyze-home-'); - const previousHome = process.env.GITNEXUS_HOME; - const previousExtensionPolicy = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; try { - process.env.GITNEXUS_HOME = home.dbPath; - process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; + vi.stubEnv('GITNEXUS_HOME', home.dbPath); + if (managedCacheRoot) vi.stubEnv('GITNEXUS_MOVE_FLOW_DIR', managedCacheRoot); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); await fs.mkdir(path.join(repo.dbPath, 'sources'), { recursive: true }); await fs.writeFile( path.join(repo.dbPath, 'Move.toml'), @@ -32,63 +110,62 @@ describe.skipIf(!canRun)('Move runFullAnalysis persistence', () => { path.join(repo.dbPath, 'helper.ts'), 'export function helper(): number { return 1; }\n', ); + initializeGitRepo(repo.dbPath); - await runFullAnalysis( - repo.dbPath, - { - force: true, - skipGit: true, - skipAgentsMd: true, - skipSkills: true, - noStats: true, - workerPoolSize: 2, - }, - { onProgress: () => {} }, - ); + await analyze(repo.dbPath, true); const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); const meta = await loadMeta(storagePath); expect(meta?.moveIngestAvailable).toBe(true); - expect(meta?.moveCompilerIdentity?.version).toMatch(/^2\./); + expect(meta?.moveCompilerIdentity).toEqual(expectedCompilerIdentity); + if (requireMoveFlow) { + expect(meta?.moveCompilerIdentity?.version).toBe(MOVE_FLOW_RELEASE.version); + } - const adapter = await import('../../src/core/lbug/lbug-adapter.js'); - await adapter.initLbug(lbugPath); + let graphBeforeFailure: GraphSnapshot; + await initLbug(lbugPath); try { await expect( - adapter.executeQuery( + executeQuery( "MATCH (f:Function) WHERE f.qualifiedName = '0x42::main::start' " + 'RETURN f.name AS name, f.isEntry AS isEntry', ), ).resolves.toEqual([{ name: 'start', isEntry: true }]); await expect( - adapter.executeQuery( + executeQuery( "MATCH (f:Function) WHERE f.name = 'helper' " + 'RETURN f.language AS language, f.qualifiedName AS qualifiedName', ), ).resolves.toEqual([{ language: 'typescript', qualifiedName: null }]); await expect( - adapter.executeQuery( + executeQuery( "MATCH (m:Module)-[r:CodeRelation]->(f:Function) WHERE r.type = 'DEFINES' " + "AND f.qualifiedName = '0x42::main::start' " + 'RETURN m.qualifiedName AS module, r.type AS type', ), ).resolves.toEqual([{ module: '0x42::main', type: 'DEFINES' }]); await expect( - adapter.executeQuery( + executeQuery( 'MATCH (n) WITH n.id AS id, count(n) AS copies ' + 'WHERE copies > 1 RETURN id, copies', ), ).resolves.toEqual([]); + graphBeforeFailure = await queryGraphSnapshot(); } finally { - await adapter.closeLbug(); + await closeLbug(); + } + + vi.stubEnv('MOVE_FLOW', path.join(repo.dbPath, 'missing-move-flow')); + await fs.appendFile(path.join(repo.dbPath, 'helper.ts'), '// compiler unavailable\n'); + + for (const force of [false, true]) { + await expect(analyze(repo.dbPath, force)).rejects.toThrow( + 'move-flow is unavailable; the existing Move graph was left unchanged', + ); + expect(await loadMeta(storagePath)).toEqual(meta); + expect(await readGraphSnapshot(lbugPath)).toEqual(graphBeforeFailure); } } finally { - if (previousHome === undefined) delete process.env.GITNEXUS_HOME; - else process.env.GITNEXUS_HOME = previousHome; - if (previousExtensionPolicy === undefined) { - delete process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; - } else { - process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = previousExtensionPolicy; - } + vi.unstubAllEnvs(); await repo.cleanup(); await home.cleanup(); } diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index 0b5473099..ba99493c3 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -191,52 +191,6 @@ describe('runFullAnalysis — incremental orchestration', () => { } }, 300_000); - it('leaves a compiler-backed Move index untouched when move-flow is unavailable', async () => { - const repo = await setupMiniRepo(); - try { - // An explicit MOVE_FLOW override that fails its probe is authoritative - // and skips auto-install — deterministic "compiler unavailable" offline. - vi.stubEnv('MOVE_FLOW', path.join(repo.dbPath, 'nonexistent-move-flow')); - await writeFile( - path.join(repo.dbPath, 'Move.toml'), - '[package]\nname = "mini"\nversion = "1.0.0"\n', - ); - gitCommitAll(repo.dbPath, 'add move package'); - - const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); - await runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }); - - const { storagePath } = getStoragePaths(repo.dbPath); - const first = await loadMeta(storagePath); - expect(first?.moveIngestAvailable).toBe(false); - if (!first) throw new Error('expected metadata after initial analysis'); - - // Simulate an index whose Move facts were built by a since-vanished - // compiler (e.g. the release cache was wiped between runs). - const identity = { version: '2.0.0', source: 'release', fingerprint: 'fp' } as const; - const compilerBackedMeta: RepoMeta = { - ...first, - moveIngestAvailable: true, - moveCompilerIdentity: identity, - }; - await saveMeta(storagePath, compilerBackedMeta); - - const target = path.join(repo.dbPath, 'src', 'logger.ts'); - await writeFile(target, (await readFile(target, 'utf-8')) + '\n// touched by test\n'); - await expect( - runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }), - ).rejects.toThrow('move-flow is unavailable; the existing Move graph was left unchanged'); - - expect(await loadMeta(storagePath)).toEqual(compilerBackedMeta); - await expect( - runFullAnalysis(repo.dbPath, { skipAgentsMd: true, force: true }, { onProgress: () => {} }), - ).rejects.toThrow('existing Move graph was left unchanged'); - expect(await loadMeta(storagePath)).toEqual(compilerBackedMeta); - } finally { - await repo.cleanup(); - } - }, 300_000); - it('incremental output is byte-equivalent to a full rebuild (incremental ≡ --force on the same repo state)', async () => { // The central correctness contract of this PR: an incremental run // and a full rebuild from the same repo state must produce identical diff --git a/gitnexus/test/unit/move/provision.test.ts b/gitnexus/test/unit/move/provision.test.ts index e56009275..f63fab627 100644 --- a/gitnexus/test/unit/move/provision.test.ts +++ b/gitnexus/test/unit/move/provision.test.ts @@ -78,30 +78,28 @@ describe('ensureMoveFlowRuntime', () => { expect(deps.resolveClient).toHaveBeenCalledTimes(1); }); - it('retries after a rejected installation', async () => { + it('does not retry a rejected installation within the same process', async () => { delete process.env.MOVE_FLOW; const install = vi .fn() - .mockRejectedValueOnce(new Error('offline')) - .mockResolvedValueOnce({ status: 'installed', binary: cached }); + .mockRejectedValue(new Error('offline')); const deps = dependencies({ install }); await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); - await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); - expect(install).toHaveBeenCalledTimes(2); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + expect(install).toHaveBeenCalledOnce(); }); - it('retries after a soft installation failure', async () => { + it('does not retry a soft installation failure within the same process', async () => { delete process.env.MOVE_FLOW; const install = vi .fn() - .mockResolvedValueOnce({ status: 'failed', message: 'checksum service unavailable' }) - .mockResolvedValueOnce({ status: 'installed', binary: cached }); + .mockResolvedValue({ status: 'failed', message: 'checksum service unavailable' }); const deps = dependencies({ install }); await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); - await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); - expect(install).toHaveBeenCalledTimes(2); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toBeNull(); + expect(install).toHaveBeenCalledOnce(); }); it('shares only an in-flight install attempt', async () => { @@ -124,6 +122,16 @@ describe('ensureMoveFlowRuntime', () => { expect(deps.createClient).toHaveBeenCalledTimes(2); }); + it('clears a successful install attempt so a later cache miss can reinstall', async () => { + delete process.env.MOVE_FLOW; + const install = vi.fn(async () => ({ status: 'installed' as const, binary: cached })); + const deps = dependencies({ install }); + + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); + await expect(ensureMoveFlowRuntime({}, deps)).resolves.toMatchObject({ client }); + expect(install).toHaveBeenCalledTimes(2); + }); + it('uses the verified cache without another version probe', async () => { delete process.env.MOVE_FLOW; const deps = dependencies({ findCached: vi.fn(async () => cached) }); diff --git a/gitnexus/test/unit/node-table-layout.test.ts b/gitnexus/test/unit/node-table-layout.test.ts index a4a4451c0..c6daa18a7 100644 --- a/gitnexus/test/unit/node-table-layout.test.ts +++ b/gitnexus/test/unit/node-table-layout.test.ts @@ -75,4 +75,24 @@ describe('shared node-table persistence layouts', () => { expect(copyColumns(getCopyQuery(table, '/tmp/fixture.csv'))).toEqual(columns); }, ); + + it('rejects an unknown CSV column encoding instead of emitting an empty cell', () => { + const column = NODE_TABLE_LAYOUTS.Function.columns[0]; + const mutableColumn = column as { csvEncoding: string }; + const originalEncoding = column.csvEncoding; + const node = { + id: 'Function:fixture', + label: 'Function', + properties: { name: 'fixture', filePath: 'src/fixture.ts' }, + } as GraphNode; + + try { + mutableColumn.csvEncoding = 'future-encoding'; + expect(() => buildLayoutNodeRow('Function', node, 'fixture content')).toThrow( + 'Unsupported CSV column encoding: future-encoding', + ); + } finally { + mutableColumn.csvEncoding = originalEncoding; + } + }); }); From b2890759f45b448e88592d6febf38023f674611a Mon Sep 17 00:00:00 2001 From: abhigyantrumio Date: Thu, 23 Jul 2026 03:32:30 +0530 Subject: [PATCH 4/4] fix(move): open cached metadata with O_NOFOLLOW instead of pre-lstat CodeQL flagged the lstat-then-open pair as a fresh check-then-use race (js/file-system-race #924). Drop the path-level probe entirely: a single open(O_RDONLY|O_NOFOLLOW) rejects a symlinked final component atomically on POSIX, and every subsequent gate reads through the handle. Windows has no O_NOFOLLOW (0 fallback); the fstat isFile() gate and the downstream metadata/binary-hash validation cover that platform's threat model. Co-Authored-By: Claude Fable 5 --- gitnexus/src/core/move/install.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/gitnexus/src/core/move/install.ts b/gitnexus/src/core/move/install.ts index 69e7d9ad0..23d12d3c5 100644 --- a/gitnexus/src/core/move/install.ts +++ b/gitnexus/src/core/move/install.ts @@ -300,13 +300,17 @@ const validCachedInstall = async ( ): Promise => { let metadataHandle: FileHandle | undefined; try { - // Symlinked metadata is rejected up front; every subsequent check reads - // through one open handle so the size gate and the JSON read cannot race - // a concurrent swap of the file (CodeQL js/file-system-race). - if (!(await lstat(config.metadataPath)).isFile()) { - return null; - } - metadataHandle = await open(config.metadataPath, 'r'); + // Single open, then every check reads through the handle, so the type/size + // gates and the JSON read cannot race a concurrent swap of the file + // (CodeQL js/file-system-race). O_NOFOLLOW makes the kernel reject a + // symlinked final component atomically on POSIX; Windows has no such flag + // (0 fallback) — there the fstat isFile() gate plus the downstream + // metadata/binary-hash validation carry the (home-dir, local-writer) + // threat model. + metadataHandle = await open( + config.metadataPath, + fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0), + ); const metadataStat = await metadataHandle.stat(); if ( !metadataStat.isFile() ||