From bf1f38b9aa7d2df5a6362c591c4038a66fa33db7 Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 30 Jun 2026 20:40:00 +0800 Subject: [PATCH 01/14] adds an opt-in auto sync and analysis loop for GitNexus --- .gitignore | 3 + gitnexus/src/cli/index.ts | 6 + gitnexus/src/core/auto-sync/config.ts | 163 +++++++ gitnexus/src/core/auto-sync/index.ts | 40 ++ gitnexus/src/core/auto-sync/path-security.ts | 173 ++++++++ gitnexus/src/core/auto-sync/repo.ts | 5 + gitnexus/src/core/auto-sync/runner.ts | 201 +++++++++ gitnexus/src/core/auto-sync/starter.ts | 51 +++ gitnexus/src/core/auto-sync/state.ts | 61 +++ gitnexus/src/server/git-clone.ts | 115 ++++- gitnexus/test/unit/auto-sync-runner.test.ts | 421 +++++++++++++++++++ gitnexus/test/unit/auto-sync.test.ts | 250 +++++++++++ gitnexus/test/unit/git-clone.test.ts | 194 ++++++++- 13 files changed, 1656 insertions(+), 27 deletions(-) create mode 100644 gitnexus/src/core/auto-sync/config.ts create mode 100644 gitnexus/src/core/auto-sync/index.ts create mode 100644 gitnexus/src/core/auto-sync/path-security.ts create mode 100644 gitnexus/src/core/auto-sync/repo.ts create mode 100644 gitnexus/src/core/auto-sync/runner.ts create mode 100644 gitnexus/src/core/auto-sync/starter.ts create mode 100644 gitnexus/src/core/auto-sync/state.ts create mode 100644 gitnexus/test/unit/auto-sync-runner.test.ts create mode 100644 gitnexus/test/unit/auto-sync.test.ts diff --git a/.gitignore b/.gitignore index 11f2743c7..bf7f113a8 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ npm-debug.log* # Testing coverage/ +.tmp-test/ +gitnexus/.tmp-test/ # Misc *.local @@ -110,3 +112,4 @@ local_docs/ .agents/ .context/ gitnexus/web/ +/log/ diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 72e4da365..4426eec9d 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -15,6 +15,12 @@ const _require = createRequire(import.meta.url); const pkg = _require('../../package.json'); const program = new Command(); +if (process.env.AUTO_UPDATE_AND_ANALYZE_FLAG?.trim() === '1') { + void import('../core/auto-sync/index.js').then(({ maybeStartAutoSyncFromEnv }) => + maybeStartAutoSyncFromEnv(), + ); +} + function collectCodingAgents(value: string, previous: string[] | undefined): string[] { return [...(previous ?? []), ...value.split(',')]; } diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts new file mode 100644 index 000000000..9c0a67e99 --- /dev/null +++ b/gitnexus/src/core/auto-sync/config.ts @@ -0,0 +1,163 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { normalizeConfiguredCloneRoot } from './path-security.js'; + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +export const AUTO_SYNC_FLAG = 'AUTO_UPDATE_AND_ANALYZE_FLAG'; +export const AUTO_SYNC_CONFIG_FILE = 'sync_config.yml'; +const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +const MIN_SYNC_INTERVAL_MINUTES = 5; + +export interface AutoSyncProjectConfig { + localPath: string; + gitnexusGroup?: string; + branches: string[]; + remoteUrls: string[]; +} + +export interface AutoSyncConfig { + configPath: string; + syncIntervalMinutes: number; + projects: AutoSyncProjectConfig[]; +} + +export type AutoSyncFlagDecision = + | { enabled: true } + | { enabled: false; reason: 'unset' | 'disabled' | 'invalid'; message?: string }; + +export type AutoSyncConfigLoadResult = + | { ok: true; config: AutoSyncConfig } + | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; + +export function parseAutoSyncFlag(raw = process.env[AUTO_SYNC_FLAG]): AutoSyncFlagDecision { + if (raw === undefined || raw.trim() === '') return { enabled: false, reason: 'unset' }; + const trimmed = raw.trim(); + if (trimmed === '0') return { enabled: false, reason: 'disabled' }; + if (trimmed === '1') return { enabled: true }; + return { + enabled: false, + reason: 'invalid', + message: `[auto-sync] ${AUTO_SYNC_FLAG} must be 0 or 1; got "${trimmed}". Auto sync is disabled.`, + }; +} + +export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); +} + +export function parseBranchCandidates(branchValue: unknown): string[] { + const rawItems = Array.isArray(branchValue) + ? branchValue.flatMap((item) => String(item).split(',')) + : String(branchValue ?? '').split(','); + const branches: string[] = []; + const seen = new Set(); + for (const item of rawItems) { + const branch = item.trim(); + if (!branch || seen.has(branch)) continue; + seen.add(branch); + branches.push(branch); + } + return branches; +} + +export async function loadAutoSyncConfig( + configPath = getAutoSyncConfigPath(), +): Promise { + let content: string; + try { + content = await fs.readFile(configPath, 'utf-8'); + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${configPath}. Auto sync is skipped.`, + }; + } + return { + ok: false, + reason: 'unreadable', + message: `[auto-sync] Unable to read config file: ${configPath}. Auto sync is skipped.`, + }; + } + + try { + return { ok: true, config: parseAutoSyncConfig(content, configPath) }; + } catch (err: unknown) { + return { + ok: false, + reason: 'invalid', + message: `[auto-sync] Invalid sync_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + }; + } +} + +export function parseAutoSyncConfig(content: string, configPath: string): AutoSyncConfig { + const raw = yaml.load(content, { schema: yaml.JSON_SCHEMA }) as Record; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('expected a YAML object'); + } + + const errors: string[] = []; + const interval = Number(raw.sync_interval_minutes); + if (!Number.isInteger(interval) || interval <= 0) { + errors.push('sync_interval_minutes must be a positive integer'); + } else if (interval < MIN_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); + } + + const rawProjects = raw.projects; + if (!Array.isArray(rawProjects) || rawProjects.length === 0) { + errors.push('projects must contain at least one project'); + } + + const projects: AutoSyncProjectConfig[] = []; + if (Array.isArray(rawProjects)) { + rawProjects.forEach((projectValue, index) => { + const project = projectValue as Record; + if (!project || typeof project !== 'object' || Array.isArray(project)) { + errors.push(`projects[${index}] must be an object`); + return; + } + + const localPath = typeof project.local_path === 'string' ? project.local_path.trim() : ''; + if (!localPath) { + errors.push(`projects[${index}].local_path is required`); + } else { + try { + normalizeConfiguredCloneRoot(localPath); + } catch (err: unknown) { + errors.push(`projects[${index}].local_path ${(err as Error).message}`); + } + } + + const remoteUrls = Array.isArray(project.remote_urls) + ? project.remote_urls.map((url) => String(url).trim()).filter(Boolean) + : []; + if (remoteUrls.length === 0) { + errors.push(`projects[${index}].remote_urls must contain at least one URL`); + } + + const branches = parseBranchCandidates(project.branch); + if (branches.length === 0) errors.push(`projects[${index}].branch is required`); + + const gitnexusGroup = + typeof project.gitnexus_group === 'string' ? project.gitnexus_group.trim() : undefined; + if (gitnexusGroup && !GROUP_NAME_PATTERN.test(gitnexusGroup)) { + errors.push(`projects[${index}].gitnexus_group is invalid`); + } + + if (localPath && remoteUrls.length > 0 && branches.length > 0) { + projects.push({ localPath, gitnexusGroup, branches, remoteUrls }); + } + }); + } + + if (errors.length > 0) throw new Error(errors.join('; ')); + return { configPath, syncIntervalMinutes: interval, projects }; +} diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts new file mode 100644 index 000000000..de1c0b4f7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/index.ts @@ -0,0 +1,40 @@ +export { + AUTO_SYNC_CONFIG_FILE, + AUTO_SYNC_FLAG, + getAutoSyncConfigPath, + loadAutoSyncConfig, + parseAutoSyncConfig, + parseAutoSyncFlag, + parseBranchCandidates, + type AutoSyncConfig, + type AutoSyncConfigLoadResult, + type AutoSyncFlagDecision, + type AutoSyncProjectConfig, +} from './config.js'; +export { + buildStateKey, + getAutoSyncStatePath, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitState, + type AutoSyncCommitStateEntry, +} from './state.js'; +export { extractRepoNameFromRemoteUrl } from './repo.js'; +export { + normalizeConfiguredCloneRoot, + quarantineAutoSyncPartial, + resolveConfiguredCloneRoot, + type AutoSyncCloneRoot, +} from './path-security.js'; +export { + addRepoToGroup, + getConfiguredRepoPath, + runAutoSyncOnce, + syncGroupByName, + type AutoSyncLogger, + type AutoSyncRunDeps, + type AutoSyncRunResult, +} from './runner.js'; +export { maybeStartAutoSyncFromEnv, type AutoSyncStartHandle } from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts new file mode 100644 index 000000000..10f5d6ab7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -0,0 +1,173 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +const DANGEROUS_ROOTS = new Set( + [ + '/', + os.homedir(), + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ].map((entry) => path.resolve(entry)), +); + +const DANGEROUS_PARENT_ROOTS = new Set( + [ + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ].map((entry) => path.resolve(entry)), +); + +const QUARANTINE_RETENTION_DAYS = 14; + +export interface AutoSyncCloneRoot { + root: string; + quarantineRoot: string; + quarantineRetentionDays: number; +} + +export async function resolveConfiguredCloneRoot(localPath: string): Promise { + const root = normalizeConfiguredCloneRoot(localPath); + assertNotDangerousRoot(root); + await assertNoSymlinkPath(root); + await assertDirectoryOwnerAndPermissions(root); + const realRoot = await fs.realpath(root); + assertContainedOrSame(root, realRoot, 'Configured clone root realpath escaped its normalized path'); + assertNotDangerousRoot(realRoot); + assertNotGitNexusInternalRoot(realRoot); + + return { + root: realRoot, + quarantineRoot: path.join(getGlobalDir(), 'quarantine'), + quarantineRetentionDays: QUARANTINE_RETENTION_DAYS, + }; +} + +export function normalizeConfiguredCloneRoot(localPath: string): string { + const value = localPath.trim(); + if (!value) throw new Error('local_path is required'); + if (!path.isAbsolute(value)) throw new Error('local_path must be an absolute path'); + if (value.split(path.sep).includes('..')) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + const resolved = path.resolve(value); + if (resolved !== path.normalize(value)) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + return resolved; +} + +export async function quarantineAutoSyncPartial(targetDir: string, quarantineRoot: string): Promise { + await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); + const base = path.basename(targetDir); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const destination = path.join(quarantineRoot, `auto-sync-${stamp}-${process.pid}-${base}`); + await fs.rename(targetDir, destination); + await fs.writeFile( + `${destination}.README.txt`, + [ + 'GitNexus auto-sync isolated a partial or unsafe clone result.', + `Created at: ${new Date().toISOString()}`, + `Original path: ${targetDir}`, + `Retention: keep for ${QUARANTINE_RETENTION_DAYS} days unless an operator reviews and removes it earlier.`, + 'Cleanup: verify the original path and remote before manual deletion.', + '', + ].join('\n'), + 'utf-8', + ); + return destination; +} + +function assertNotDangerousRoot(root: string): void { + if (DANGEROUS_ROOTS.has(root)) throw new Error(`Refusing unsafe auto-sync clone root: ${root}`); + for (const dangerousRoot of DANGEROUS_PARENT_ROOTS) { + const rel = path.relative(dangerousRoot, root); + if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) { + throw new Error(`Refusing unsafe auto-sync clone root under ${dangerousRoot}: ${root}`); + } + } + if (path.parse(root).root === root) throw new Error(`Refusing filesystem root as clone root: ${root}`); +} + +function assertNotGitNexusInternalRoot(root: string): void { + const gitnexusDir = path.resolve(getGlobalDir()); + const blocked = [ + gitnexusDir, + path.join(gitnexusDir, 'groups'), + path.join(gitnexusDir, 'indexes'), + path.join(gitnexusDir, 'quarantine'), + ]; + for (const blockedRoot of blocked) { + const rel = path.relative(blockedRoot, root); + if (!rel || (!rel.startsWith('..') && !path.isAbsolute(rel))) { + throw new Error(`Refusing GitNexus internal directory as auto-sync clone root: ${root}`); + } + } +} + +async function assertNoSymlinkPath(root: string): Promise { + const parsed = path.parse(root); + let current = parsed.root; + const parts = root.slice(parsed.root.length).split(path.sep).filter(Boolean); + for (const part of parts) { + current = path.join(current, part); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); + } +} + +export async function assertDirectoryOwnerAndPermissions(root: string): Promise { + const stat = await fs.stat(root); + if (!stat.isDirectory()) throw new Error(`auto-sync clone root is not a directory: ${root}`); + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`auto-sync clone root is owned by uid ${stat.uid}, not current process uid`); + } + const mode = stat.mode & 0o777; + const worldWritable = (mode & 0o002) !== 0; + const sticky = (stat.mode & 0o1000) !== 0; + if (worldWritable && !sticky) { + throw new Error(`Refusing world-writable auto-sync clone root without sticky bit: ${root}`); + } +} + +function assertContainedOrSame(root: string, child: string, message: string): void { + const rel = path.relative(root, child); + if (rel.startsWith('..') || path.isAbsolute(rel)) throw new Error(message); +} diff --git a/gitnexus/src/core/auto-sync/repo.ts b/gitnexus/src/core/auto-sync/repo.ts new file mode 100644 index 000000000..76dde2e49 --- /dev/null +++ b/gitnexus/src/core/auto-sync/repo.ts @@ -0,0 +1,5 @@ +import { extractRepoName } from '../../server/git-clone.js'; + +export function extractRepoNameFromRemoteUrl(remoteUrl: string): string { + return extractRepoName(remoteUrl); +} diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts new file mode 100644 index 000000000..c5e2232e4 --- /dev/null +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -0,0 +1,201 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import yaml from 'js-yaml'; +import { loadGroupConfig } from '../group/config-parser.js'; +import { getDefaultGitnexusDir, getGroupDir } from '../group/storage.js'; +import { syncGroup } from '../group/sync.js'; +import { runFullAnalysis } from '../run-analyze.js'; +import { getCurrentBranch, getCurrentCommit } from '../../storage/git.js'; +import { registerRepo, type RepoMeta } from '../../storage/repo-manager.js'; +import { extractRepoNameFromRemoteUrl } from './repo.js'; +import { cloneOrPull } from '../../server/git-clone.js'; +import { resolveConfiguredCloneRoot } from './path-security.js'; +import { + buildStateKey, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + type AutoSyncAnalyzeStatus, +} from './state.js'; +import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js'; + +export interface AutoSyncLogger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export interface AutoSyncRunDeps { + cloneOrPull: typeof cloneOrPull; + getCurrentBranch: typeof getCurrentBranch; + getCurrentCommit: typeof getCurrentCommit; + runFullAnalysis: typeof runFullAnalysis; + registerRepo: typeof registerRepo; + loadState: typeof loadAutoSyncState; + saveState: typeof saveAutoSyncState; + addRepoToGroup: typeof addRepoToGroup; + syncGroupByName: typeof syncGroupByName; + resolveCloneRoot: typeof resolveConfiguredCloneRoot; +} + +export interface AutoSyncRunResult { + synced: number; + analyzed: number; + skippedAnalysis: number; + failed: number; +} + +const DEFAULT_LOGGER: AutoSyncLogger = { + info: (message) => process.stderr.write(`${message}\n`), + warn: (message) => process.stderr.write(`${message}\n`), + error: (message) => process.stderr.write(`${message}\n`), +}; + +const DEFAULT_DEPS: AutoSyncRunDeps = { + cloneOrPull, + getCurrentBranch, + getCurrentCommit, + runFullAnalysis, + registerRepo, + loadState: loadAutoSyncState, + saveState: saveAutoSyncState, + addRepoToGroup, + syncGroupByName, + resolveCloneRoot: resolveConfiguredCloneRoot, +}; + +export async function runAutoSyncOnce( + config: AutoSyncConfig, + options: { deps?: Partial; logger?: AutoSyncLogger; now?: () => Date } = {}, +): Promise { + const deps = { ...DEFAULT_DEPS, ...options.deps }; + const logger = options.logger ?? DEFAULT_LOGGER; + const now = options.now ?? (() => new Date()); + const state = await deps.loadState(); + const groupsToSync = new Set(); + const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + + for (const project of config.projects) { + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const cloneRoot = await deps.resolveCloneRoot(project.localPath); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); + await deps.cloneOrPull(remoteUrl, targetDir, undefined, { + allowedCloneRoot: cloneRoot.root, + expectedRepoName: repoName, + quarantineRoot: cloneRoot.quarantineRoot, + }); + result.synced += 1; + + const currentBranch = deps.getCurrentBranch(targetDir); + if (!currentBranch || !project.branches.includes(currentBranch)) { + result.skippedAnalysis += 1; + logger.warn( + `[auto-sync] Skip analysis for ${targetDir}; current branch ${currentBranch ?? ''} is not in configured branches: ${project.branches.join(', ')}.`, + ); + continue; + } + + const branch = currentBranch; + const currentCommit = deps.getCurrentCommit(targetDir); + const stateKey = buildStateKey(targetDir, branch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + + if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runFullAnalysis( + targetDir, + { branch, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + const meta: RepoMeta = { + repoPath: targetDir, + lastCommit: currentCommit, + indexedAt: now().toISOString(), + stats: analysis.stats, + branch, + }; + await deps.registerRepo(targetDir, meta, { name: repoName, allowDuplicateName: true }); + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + result.analyzed += 1; + } catch (err: unknown) { + analyzeStatus = 'failed'; + result.failed += 1; + logger.error(`[auto-sync] Analysis failed for ${targetDir}: ${(err as Error).message}`); + } + } else { + result.skippedAnalysis += 1; + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + + state[stateKey] = { + codeCommitId: currentCommit, + analyzedCommitId, + lastAnalyzeStatus: analyzeStatus, + lastSyncTime: now().toISOString(), + }; + + if (project.gitnexusGroup) { + const added = await deps.addRepoToGroup(project, repoName); + if (added) groupsToSync.add(project.gitnexusGroup); + } + } catch (err: unknown) { + result.failed += 1; + logger.error(`[auto-sync] Repository sync failed for ${remoteUrl}: ${(err as Error).message}`); + } + } + } + + await deps.saveState(state); + for (const groupName of groupsToSync) { + try { + await deps.syncGroupByName(groupName); + } catch (err: unknown) { + result.failed += 1; + logger.error(`[auto-sync] Group sync failed for ${groupName}: ${(err as Error).message}`); + } + } + return result; +} + +export function getConfiguredRepoPath( + project: Pick, + repoName: string, +): string { + return path.resolve(project.localPath, repoName); +} + +export async function addRepoToGroup( + project: Pick, + repoName: string, +): Promise { + if (!project.gitnexusGroup) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.gitnexusGroup); + const config = await loadGroupConfig(groupDir); + if (Object.values(config.repos).includes(repoName)) return false; + config.repos[repoName] = repoName; + await writeGroupConfigAtomic(path.join(groupDir, 'group.yaml'), config); + return true; +} + +export async function syncGroupByName(groupName: string): Promise { + const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName); + const config = await loadGroupConfig(groupDir); + await syncGroup(config, { groupDir, allowStale: true }); +} + +async function writeGroupConfigAtomic(filePath: string, config: unknown): Promise { + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); + await fs.rename(tmpPath, filePath); +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts new file mode 100644 index 000000000..579b18ab5 --- /dev/null +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -0,0 +1,51 @@ +import { loadAutoSyncConfig, parseAutoSyncFlag } from './config.js'; +import { runAutoSyncOnce } from './runner.js'; + +export interface AutoSyncStartHandle { + stop(): void; +} + +export async function maybeStartAutoSyncFromEnv(options: { + setIntervalFn?: typeof setInterval; + clearIntervalFn?: typeof clearInterval; + runOnce?: typeof runAutoSyncOnce; + stderr?: Pick; +} = {}): Promise { + const stderr = options.stderr ?? process.stderr; + const flag = parseAutoSyncFlag(); + if (flag.enabled === false) { + if (flag.message) stderr.write(`${flag.message}\n`); + return null; + } + + const loaded = await loadAutoSyncConfig(); + if (loaded.ok === false) { + stderr.write(`${loaded.message}\n`); + return null; + } + + const runOnce = options.runOnce ?? runAutoSyncOnce; + let running = false; + const runSafely = () => { + if (running) { + stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); + return; + } + running = true; + void runOnce(loaded.config) + .catch((err: unknown) => { + stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + }) + .finally(() => { + running = false; + }); + }; + + runSafely(); + const intervalMs = loaded.config.syncIntervalMinutes * 60_000; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; + const timer = setIntervalFn(runSafely, intervalMs); + timer.unref?.(); + return { stop: () => clearIntervalFn(timer) }; +} diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts new file mode 100644 index 000000000..f14d9aa5f --- /dev/null +++ b/gitnexus/src/core/auto-sync/state.ts @@ -0,0 +1,61 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped'; + +export interface AutoSyncCommitStateEntry { + codeCommitId: string; + analyzedCommitId?: string; + lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + lastSyncTime: string; +} + +export type AutoSyncCommitState = Record; + +export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'auto-sync-state.json'); +} + +export function buildStateKey(repoPath: string, branch: string): string { + return `${path.resolve(repoPath)}|${branch}`; +} + +export function shouldAnalyzeCommit(input: { + currentCommit: string; + previousAnalyzedCommit?: string; + previousStatus?: AutoSyncAnalyzeStatus; +}): boolean { + if (!input.currentCommit) return false; + if (input.previousStatus === 'failed') return true; + return input.currentCommit !== input.previousAnalyzedCommit; +} + +export async function loadAutoSyncState( + statePath = getAutoSyncStatePath(), +): Promise { + try { + const raw = await fs.readFile(statePath, 'utf-8'); + const parsed = JSON.parse(raw); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as AutoSyncCommitState) + : {}; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + process.stderr.write( + `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + } + return {}; + } +} + +export async function saveAutoSyncState( + state: AutoSyncCommitState, + statePath = getAutoSyncStatePath(), +): Promise { + await fs.mkdir(path.dirname(statePath), { recursive: true }); + const tmpPath = `${statePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, statePath); +} diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 477f47a6e..588f36ea3 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -10,8 +10,11 @@ import path from 'path'; import fs from 'fs/promises'; import { isIP } from 'net'; import { logger } from '../core/logger.js'; -import { parseRepoNameFromUrl } from '../storage/git.js'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { + assertDirectoryOwnerAndPermissions, + quarantineAutoSyncPartial, +} from '../core/auto-sync/path-security.js'; /** * Root directory for all cloned repositories. Targets must resolve inside this. @@ -39,12 +42,16 @@ export const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; * clone root via path traversal. */ export function extractRepoName(url: string): string { - const name = parseRepoNameFromUrl(url); + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const name = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; if ( !name || name === '.' || name === '..' || name === 'unknown' || + name.startsWith('-') || !REPO_NAME_PATTERN.test(name) ) { throw new Error('Could not extract a valid repository name from URL'); @@ -235,6 +242,14 @@ export interface CloneProgress { message: string; } +export interface CloneOrPullOptions { + token?: string; + allowedCloneRoot?: string; + expectedRepoName?: string; + quarantineRoot?: string; + runGitForTest?: typeof runGit; +} + /** * Build the `git clone` argument list for a given URL and target directory. * @@ -443,36 +458,54 @@ export async function cloneOrPull( url: string, targetDir: string, onProgress?: (progress: CloneProgress) => void, - options?: { token?: string }, + options?: CloneOrPullOptions, ): Promise { // Containment barrier — inline with the canonical path.relative idiom so // CodeQL recognizes the sanitizer at every following filesystem and // subprocess sink. The same `safeTarget` is used for every downstream // path operation — no reassignment that the analyzer could lose track of. // - // Limitation: this is a lexical containment check, not a realpath check. - // If an attacker can place a symlink under CLONE_ROOT pointing outside it, - // the lexical check passes but the clone lands at the symlink target. That - // requires pre-existing local write access to CLONE_ROOT, so the threat - // model considers it out of scope; CodeQL js/path-injection accepts the - // lexical form. Tracked as a follow-up if defense-in-depth is needed. + // The lexical check runs before filesystem creation; realpath and symlink + // checks below run before pull/clone and again after clone completes. + const cloneRoot = path.resolve(options?.allowedCloneRoot ?? CLONE_ROOT); + const expectedRepoName = options?.expectedRepoName; + if (expectedRepoName !== undefined && expectedRepoName !== extractRepoName(url)) { + throw new Error(`Clone target repo name ${expectedRepoName} does not match requested URL`); + } + const safeTarget = path.resolve(targetDir); - const rel = path.relative(CLONE_ROOT, safeTarget); + if (expectedRepoName !== undefined && path.basename(safeTarget) !== expectedRepoName) { + throw new Error(`Clone target basename must match repository name ${expectedRepoName}`); + } + + const rel = path.relative(cloneRoot, safeTarget); if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { - throw new Error(`Clone target must be a subdirectory of ${CLONE_ROOT}`); + throw new Error(`Clone target must be a subdirectory of ${cloneRoot}`); } // Always validate the requested URL — the prior shape only ran this in // the code path where the repo was cloned. Now it runs unconditionally, // preventing SSRF / blocked-host bypasses even when targetDir already exists. validateGitUrl(url); + await fs.mkdir(cloneRoot, { recursive: true }); + if (options?.allowedCloneRoot) { + await assertDirectoryOwnerAndPermissions(cloneRoot); + } + await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertPreRealpathContainment(cloneRoot, safeTarget); const exists = await fs.access(path.join(safeTarget, '.git')).then( () => true, () => false, ); + const targetExists = await fs.access(safeTarget).then( + () => true, + () => false, + ); + if (exists) { + await assertPostRealpathContainment(cloneRoot, safeTarget); // Confirm the existing clone is actually the same repository the caller // requested. Without this check, a pull would silently succeed against // whatever remote the dir was originally cloned from. @@ -480,14 +513,72 @@ export async function cloneOrPull( onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url }); } else { + if (targetExists) { + throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); + } await fs.mkdir(path.dirname(safeTarget), { recursive: true }); + await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertPreRealpathContainment(cloneRoot, safeTarget); onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); - await runGit(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + try { + const runGitImpl = options?.runGitForTest ?? runGit; + await runGitImpl(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + await assertPostRealpathContainment(cloneRoot, safeTarget); + } catch (err: unknown) { + if (options?.quarantineRoot) { + await fs + .access(safeTarget) + .then(async () => { + await quarantineAutoSyncPartial(safeTarget, options.quarantineRoot!); + }) + .catch(() => {}); + } + throw err; + } } return safeTarget; } +async function assertPreRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realParent = await fs.realpath(path.dirname(target)); + const parentRel = path.relative(realRoot, realParent); + if (parentRel.startsWith('..') || path.isAbsolute(parentRel)) { + throw new Error(`Clone target parent must resolve inside ${root}`); + } +} + +async function assertPostRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realTarget = await fs.realpath(target); + const rel = path.relative(realRoot, realTarget); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { + throw new Error(`Clone target must resolve inside ${root}`); + } +} + +async function assertNoSymlinkPath(root: string, target: string): Promise { + const resolvedRoot = path.resolve(root); + const resolvedTarget = path.resolve(target); + const relativeTarget = path.relative(resolvedRoot, resolvedTarget); + if (relativeTarget.startsWith('..') || path.isAbsolute(relativeTarget)) return; + let current = resolvedRoot; + for (const segment of relativeTarget.split(path.sep).filter(Boolean)) { + current = path.join(current, segment); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlink in clone target path: ${current}`); + } + } +} + /** * Hosts the per-request GitHub PAT may be sent to. Exported so the * /api/analyze boundary check and this injection-site check share one diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts new file mode 100644 index 000000000..59e358c6a --- /dev/null +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -0,0 +1,421 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; + +import { + addRepoToGroup, + getConfiguredRepoPath, + maybeStartAutoSyncFromEnv, + runAutoSyncOnce, +} from '../../src/core/auto-sync/index.js'; +import type { AutoSyncConfig, AutoSyncRunDeps } from '../../src/core/auto-sync/index.js'; + +const config: AutoSyncConfig = { + configPath: '/tmp/.gitnexus/sync_config.yml', + syncIntervalMinutes: 10, + projects: [ + { + localPath: '/tmp/repos', + gitnexusGroup: 'back_end', + branches: ['master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }, + ], +}; + +const cloneRoot = { + root: '/tmp/repos', + quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRetentionDays: 14, +}; + +function withCloneRoot(deps: Partial): Partial { + return { + resolveCloneRoot: vi.fn(async () => cloneRoot), + ...deps, + }; +} + +describe('auto-sync runner', () => { + it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/quarantine', + }, + ); + expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); + expect(deps.runFullAnalysis).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + { branch: 'master', skipAgentsMd: true, skipSkills: true }, + { onProgress: expect.any(Function) }, + ); + expect(deps.registerRepo).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + expect.objectContaining({ lastCommit: 'commit-2', branch: 'master' }), + { name: 'qts_account', allowDuplicateName: true }, + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('skips analysis when commit id has not changed', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-1'), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(0); + expect(result.skippedAnalysis).toBe(1); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.syncGroupByName).not.toHaveBeenCalled(); + }); + + it('uses local_path plus repo name as the clone target', async () => { + expect(getConfiguredRepoPath(config.projects[0], 'qts_account')).toBe('/tmp/repos/qts_account'); + + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/quarantine', + }, + ); + }); + + it('skips analysis when the checked out branch is not configured', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch develop is not in configured branches: master.', + ); + }); + + it('skips analysis when the checked out repository is detached', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => undefined), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch is not in configured branches: master.', + ); + }); + + it('isolates repository, analysis, and group sync failures', async () => { + const errorLogger = vi.fn(); + const failingConfig: AutoSyncConfig = { + ...config, + projects: [ + { + ...config.projects[0], + remoteUrls: [ + 'git@gitee.com:qts_server/failing_sync.git', + 'git@gitee.com:qts_server/qts_account.git', + ], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + if (remoteUrl.includes('failing_sync')) throw new Error('sync failed'); + return '/tmp/repos/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => { + throw new Error('analysis failed'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => { + throw new Error('group sync failed'); + }), + }); + + const result = await runAutoSyncOnce(failingConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 3 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.registerRepo).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(failingConfig.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + codeCommitId: 'commit-2', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Repository sync failed for git@gitee.com:qts_server/failing_sync.git'), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Group sync failed for back_end'), + ); + }); + + it('detects existing groupPath to registryName mappings as already joined', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-group-')); + try { + process.env.GITNEXUS_HOME = tempDir; + const groupDir = path.join(tempDir, 'groups', 'back_end'); + await fs.mkdir(groupDir, { recursive: true }); + await fs.writeFile( + path.join(groupDir, 'group.yaml'), + [ + 'version: 1', + 'name: back_end', + 'repos:', + ' hr/hiring/backend: qts_account', + ].join('\n'), + ); + + await expect(addRepoToGroup({ gitnexusGroup: 'back_end' }, 'qts_account')).resolves.toBe( + false, + ); + + await expect(fs.readFile(path.join(groupDir, 'group.yaml'), 'utf-8')).resolves.toContain( + 'hr/hiring/backend: qts_account', + ); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); + +describe('auto-sync starter', () => { + it('does not read config or register timers when the flag is disabled', async () => { + const previous = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '0'; + const setIntervalFn = vi.fn() as unknown as typeof setInterval; + + try { + const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn }); + expect(handle).toBeNull(); + expect(setIntervalFn).not.toHaveBeenCalled(); + } finally { + if (previous === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previous; + } + }); + + it('registers a clearable timer when enabled with a valid config', async () => { + const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + const runOnce = vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); + + try { + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'sync_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' gitnexus_group: back_end', + ' branch: master', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn, clearIntervalFn, runOnce }); + + expect(handle).not.toBeNull(); + expect(runOnce).toHaveBeenCalledTimes(1); + expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); + expect(timer.unref).toHaveBeenCalled(); + + handle?.stop(); + + expect(clearIntervalFn).toHaveBeenCalledWith(timer); + } finally { + if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('skips overlapping scheduled runs while a previous run is active', async () => { + const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + let scheduled: (() => void) | undefined; + const setIntervalFn = vi.fn((fn: () => void) => { + scheduled = fn; + return timer; + }) as unknown as typeof setInterval; + const stderr = { write: vi.fn() }; + let releaseRun: (() => void) | undefined; + const runOnce = vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }), + ); + + try { + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'sync_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - https://example.com/team/repo.git', + ].join('\n'), + ); + + await maybeStartAutoSyncFromEnv({ setIntervalFn, runOnce, stderr }); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(1); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Previous run is still active; skipping overlapping run.\n', + ); + + releaseRun?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(2); + } finally { + if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts new file mode 100644 index 000000000..6cc636ba6 --- /dev/null +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -0,0 +1,250 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + extractRepoNameFromRemoteUrl, + loadAutoSyncConfig, + parseAutoSyncFlag, + parseBranchCandidates, + resolveConfiguredCloneRoot, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, +} from '../../src/core/auto-sync/index.js'; + +describe('auto-sync', () => { + let tempDir: string; + let gitnexusHome: string; + let oldHome: string | undefined; + let oldFlag: string | undefined; + + beforeEach(async () => { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + tempDir = await fs.realpath(await fs.mkdtemp(path.join(base, 'gitnexus-auto-sync-'))); + gitnexusHome = path.join(tempDir, '.gitnexus'); + await fs.mkdir(gitnexusHome); + oldHome = process.env.GITNEXUS_HOME; + oldFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + process.env.GITNEXUS_HOME = gitnexusHome; + delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + }); + + afterEach(async () => { + if (oldHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = oldHome; + if (oldFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = oldFlag; + await fs.rm(tempDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('keeps auto sync disabled when the flag is unset or 0', () => { + expect(parseAutoSyncFlag(undefined)).toEqual({ enabled: false, reason: 'unset' }); + expect(parseAutoSyncFlag('0')).toEqual({ enabled: false, reason: 'disabled' }); + }); + + it('enables auto sync only for the explicit value 1', () => { + expect(parseAutoSyncFlag('1')).toEqual({ enabled: true }); + expect(parseAutoSyncFlag('true')).toEqual({ + enabled: false, + reason: 'invalid', + message: '[auto-sync] AUTO_UPDATE_AND_ANALYZE_FLAG must be 0 or 1; got "true". Auto sync is disabled.', + }); + }); + + it('loads sync_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'sync_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' gitnexus_group: back_end', + ' branch: test, master, test', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config to load'); + expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'sync_config.yml')); + expect(loaded.config.syncIntervalMinutes).toBe(10); + expect(loaded.config.projects[0]).toMatchObject({ + localPath: '/tmp/repos', + gitnexusGroup: 'back_end', + branches: ['test', 'master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }); + }); + + it('reports missing config without throwing', async () => { + const loaded = await loadAutoSyncConfig(); + + expect(loaded).toEqual({ + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'sync_config.yml')}. Auto sync is skipped.`, + }); + }); + + it('reports invalid config without throwing', async () => { + await fs.writeFile(path.join(gitnexusHome, 'sync_config.yml'), 'projects: []\n'); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.reason).toBe('invalid'); + expect(loaded.message).toContain('[auto-sync] Invalid sync_config.yml:'); + expect(loaded.message).toContain('sync_interval_minutes must be a positive integer'); + expect(loaded.message).toContain('projects must contain at least one project'); + }); + + it('rejects missing, relative, and traversal local_path values at config load', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'sync_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: ../repos', + ' branch: master', + ' remote_urls:', + ' - https://example.com/team/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('local_path must be an absolute path'); + }); + + it('hard-fails unsafe configured clone roots', async () => { + await expect(resolveConfiguredCloneRoot('/')).rejects.toThrow('unsafe auto-sync clone root'); + await expect(resolveConfiguredCloneRoot(os.homedir())).rejects.toThrow('unsafe auto-sync clone root'); + await expect(resolveConfiguredCloneRoot(path.join(await fs.realpath(os.tmpdir()), 'repos'))).rejects.toThrow( + 'unsafe auto-sync clone root', + ); + const root = path.join(tempDir, 'repos'); + await expect(resolveConfiguredCloneRoot(`${root}/../repos`)).rejects.toThrow( + 'normalized', + ); + }); + + it('rejects GitNexus internal directory descendants as clone roots', async () => { + for (const internalDir of ['groups', 'indexes', 'quarantine']) { + const root = path.join(gitnexusHome, internalDir, 'repo-root'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('GitNexus internal directory'); + } + }); + + it('rejects symlinks in configured clone root paths', async () => { + const realRoot = path.join(tempDir, 'real-root'); + const linkRoot = path.join(tempDir, 'link-root'); + await fs.mkdir(realRoot); + await fs.symlink(realRoot, linkRoot); + + await expect(resolveConfiguredCloneRoot(linkRoot)).rejects.toThrow('symlink'); + }); + + it('resolves safe configured clone roots and reports quarantine retention', async () => { + const root = path.join(tempDir, 'repos'); + await fs.mkdir(root); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'quarantine'), + quarantineRetentionDays: 14, + }), + ); + }); + + it('parses branch strings and arrays with trimming and de-duplication', () => { + expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']); + expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']); + }); + + it('extracts safe repository names from remote URLs', () => { + expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( + 'qts_account', + ); + expect(extractRepoNameFromRemoteUrl('https://example.com/team/repo-name.git')).toBe( + 'repo-name', + ); + }); + + it('rejects unsafe repository names without sanitizing them', () => { + expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/repo$name.git')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/..')).toThrow( + 'valid repository name', + ); + }); + + it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => { + expect(shouldAnalyzeCommit({ currentCommit: 'abc', previousAnalyzedCommit: 'abc' })).toBe( + false, + ); + expect( + shouldAnalyzeCommit({ + currentCommit: 'abc', + previousAnalyzedCommit: 'abc', + previousStatus: 'failed', + }), + ).toBe(true); + expect(shouldAnalyzeCommit({ currentCommit: 'def', previousAnalyzedCommit: 'abc' })).toBe( + true, + ); + }); + + it('saves state atomically and reloads it', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + + await saveAutoSyncState( + { + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }, + statePath, + ); + + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('auto-sync-state.json.tmp'), + ); + await expect(loadAutoSyncState(statePath)).resolves.toEqual({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }); + }); + + it('returns empty state and reports corrupt state files', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + await fs.writeFile(statePath, '{not-json', 'utf-8'); + + await expect(loadAutoSyncState(statePath)).resolves.toEqual({}); + + expect(stderr).toHaveBeenCalledWith( + `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + }); +}); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 4db09c837..9d879a22a 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -42,6 +42,12 @@ import { getGlobalDir } from '../../src/storage/repo-manager.js'; // load, the same point CLONE_ROOT is frozen, so the two always agree. const EXPECTED_CLONE_ROOT = path.resolve(path.join(getGlobalDir(), 'repos')); +async function mkControlledRoot(prefix: string): Promise { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + return fs.realpath(await fs.mkdtemp(path.join(base, prefix))); +} + describe('git-clone', () => { describe('extractRepoName', () => { it('extracts name from HTTPS URL', () => { @@ -95,29 +101,39 @@ describe('git-clone', () => { expect(elapsedMs).toBeLessThan(500); }); - it('strips leading dashes to prevent argument injection', () => { - expect(extractRepoName('https://github.com/user/--upload-pack=payload.git')).toBe( - 'upload-pack_payload', + it('rejects leading dashes to prevent argument injection', () => { + expect(() => extractRepoName('https://github.com/user/--upload-pack=payload.git')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://github.com/user/-repo')).toThrow( + 'valid repository name', ); - expect(extractRepoName('https://github.com/user/-repo')).toBe('repo'); }); - it('sanitizes unsafe directory characters', () => { - // sanitizeRepoName turns into _tag_ - expect(extractRepoName('https://github.com/user/repo.git')).toBe('repo_tag_'); + it('rejects unsafe directory characters instead of sanitizing them', () => { + expect(() => extractRepoName('https://github.com/user/repo.git')).toThrow( + 'valid repository name', + ); }); - it('sanitizes shell metacharacters in URL segments', () => { + it('rejects shell metacharacters in URL segments', () => { // The split on /[/:]/ does not split on backslashes or other shell chars, - // so a name like `repo;rm -rf /` would slip through without the pattern. - // After fix/sanitize-repo-name, these are sanitized to underscores. - expect(extractRepoName('https://example.com/foo:repo;rm')).toBe('repo_rm'); - expect(extractRepoName('https://example.com/foo:repo$x')).toBe('repo_x'); + // so a name like `repo;rm -rf /` must fail instead of being rewritten. + expect(() => extractRepoName('https://example.com/foo:repo;rm')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo$x')).toThrow( + 'valid repository name', + ); }); - it('sanitizes whitespace and backslashes', () => { - expect(extractRepoName('https://example.com/foo:repo name')).toBe('repo_name'); - expect(extractRepoName('https://example.com/foo:repo\\name')).toBe('repo_name'); + it('rejects whitespace and backslashes', () => { + expect(() => extractRepoName('https://example.com/foo:repo name')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo\\name')).toThrow( + 'valid repository name', + ); }); }); @@ -534,6 +550,154 @@ describe('git-clone', () => { 'Only https:// and http://', ); }); + + it('allows an explicitly controlled auto-sync clone root outside the default root', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + const target = path.join(root, 'repo'); + await expect( + cloneOrPull('http://127.0.0.1/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('private/internal'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled-root target names that do not match the remote repo name', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'other'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('basename must match'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects symlink children before clone or pull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const outside = await mkControlledRoot('gitnexus-outside-'); + try { + await fs.symlink(outside, path.join(root, 'repo')); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('symlink'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects existing clones whose remote origin mismatches the requested URL', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + try { + await new Promise((resolve, reject) => { + const proc = spawn('git', ['init'], { cwd: root, stdio: 'ignore' }); + proc.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`git init ${code}`)))); + proc.on('error', reject); + }); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')).catch(async () => { + await fs.mkdir(target); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')); + }); + await fs.writeFile( + path.join(target, '.git', 'config'), + [ + '[remote "origin"]', + '\turl = https://example.com/other/repo.git', + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '', + ].join('\n'), + ); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('not the requested URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('quarantines partial auto-sync clone output on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + runGitForTest: async () => { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'partial.txt'), 'partial', 'utf-8'); + throw new Error('git clone failed (exit code 128)'); + }, + }), + ).rejects.toThrow('git clone failed'); + + const entries = await fs.readdir(quarantineRoot); + expect(entries.some((entry) => entry.startsWith('auto-sync-') && entry.endsWith('-repo'))).toBe( + true, + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('does not quarantine an existing non-git directory on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'user-file.txt'), 'keep me', 'utf-8'); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + }), + ).rejects.toThrow('already exists but is not a git repository'); + + await expect(fs.readFile(path.join(target, 'user-file.txt'), 'utf-8')).resolves.toBe( + 'keep me', + ); + await expect(fs.access(quarantineRoot)).rejects.toThrow(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled clone roots with unsafe permissions inside cloneOrPull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await fs.chmod(root, 0o777); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(root, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); }); describe('isAzureDevOpsUrl', () => { From fbde7b3e0270ad8007f7b6e01b11c813054699e0 Mon Sep 17 00:00:00 2001 From: weiyf Date: Wed, 1 Jul 2026 18:15:45 +0800 Subject: [PATCH 02/14] adds an opt-in auto sync and analysis loop for GitNexus,gitnexus watch [init|start|restart|stop|status] --- README.md | 22 + gitnexus/README.md | 22 + gitnexus/src/cli/help-i18n.ts | 1 + gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 2 + gitnexus/src/cli/index.ts | 25 +- gitnexus/src/core/auto-sync/config.ts | 136 ++- gitnexus/src/core/auto-sync/index.ts | 21 +- gitnexus/src/core/auto-sync/path-security.ts | 6 +- gitnexus/src/core/auto-sync/repo.ts | 2 + gitnexus/src/core/auto-sync/runner.ts | 363 ++++++-- gitnexus/src/core/auto-sync/starter.ts | 448 ++++++++- gitnexus/src/core/auto-sync/state.ts | 57 +- gitnexus/src/server/api.ts | 4 +- gitnexus/src/server/git-clone.ts | 105 ++- gitnexus/test/unit/auto-sync-runner.test.ts | 933 ++++++++++++++++++- gitnexus/test/unit/auto-sync.test.ts | 245 ++++- gitnexus/test/unit/cli-index-help.test.ts | 42 +- gitnexus/test/unit/git-clone.test.ts | 130 +++ 19 files changed, 2336 insertions(+), 230 deletions(-) diff --git a/README.md b/README.md index f19ad996c..dbb203b64 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16, auto-sized to the repo). 0 is rejected — there is no sequential mode. +gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus list # List all indexed repositories @@ -262,6 +263,27 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +### `gitnexus watch` + +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + group_name: back_end + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped by `floor(availableMemoryGB / 2)` with a minimum of `1`, printed at each loop start. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group processing. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. diff --git a/gitnexus/README.md b/gitnexus/README.md index 62a329b75..2f30a5e84 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -169,6 +169,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) +gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus index # Register an existing .gitnexus/ folder into the global registry @@ -197,6 +198,27 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +### `gitnexus watch` + +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + group_name: back_end + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project. `GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `GITNEXUS_HOME/watch/auto-sync-state.json`. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index def76601b..f947c364c 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -13,6 +13,7 @@ const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', uninstall: 'help.command.uninstall.description', + watch: 'help.command.watch.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', serve: 'help.command.serve.description', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index caf61ab83..6144ef7b1 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -120,6 +120,8 @@ export const en = { 'One-time setup: configure MCP for Cursor, Claude Code, OpenCode, Codex', 'help.command.uninstall.description': 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + 'help.command.watch.description': + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 01abeedc1..627aae178 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -122,6 +122,8 @@ export const zhCN = { 'help.command.setup.description': '一次性设置：为 Cursor、Claude Code、OpenCode、Codex 配置 MCP', 'help.command.uninstall.description': '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', + 'help.command.watch.description': + '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 4426eec9d..37a527728 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -15,12 +15,6 @@ const _require = createRequire(import.meta.url); const pkg = _require('../../package.json'); const program = new Command(); -if (process.env.AUTO_UPDATE_AND_ANALYZE_FLAG?.trim() === '1') { - void import('../core/auto-sync/index.js').then(({ maybeStartAutoSyncFromEnv }) => - maybeStartAutoSyncFromEnv(), - ); -} - function collectCodingAgents(value: string, previous: string[] | undefined): string[] { return [...(previous ?? []), ...value.split(',')]; } @@ -47,6 +41,25 @@ program .option('-f, --force', 'Apply the changes (default is a dry-run preview)') .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); +program + .command('watch [action]') + .description( + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + ) + .addHelpText( + 'after', + [ + '', + 'Actions: init, start (default), restart, stop, status', + 'Configuration: GITNEXUS_HOME/watch_config.yml', + 'Runtime files: GITNEXUS_HOME/watch/watch.pid, watch.lock, watch.status.json, auto-sync-state.json', + 'Writes: GITNEXUS_HOME/watch/project_commit_info.txt', + 'Remote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.', + 'Runs once immediately, then repeats on sync_interval_minutes.', + ].join('\n'), + ) + .action(createLazyAction(() => import('./watch.js'), 'watchCommand')); + // Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook // before it overwrites the var, so the postAction hook can restore it. The // analyzeCommand env snapshot is taken AFTER this hook runs, so it cannot undo diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 9c0a67e99..febf8490f 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -7,14 +7,18 @@ import { normalizeConfiguredCloneRoot } from './path-security.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -export const AUTO_SYNC_FLAG = 'AUTO_UPDATE_AND_ANALYZE_FLAG'; -export const AUTO_SYNC_CONFIG_FILE = 'sync_config.yml'; +export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml'; const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; const MIN_SYNC_INTERVAL_MINUTES = 5; +const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_CONCURRENCY = 1; +export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3; +const MIN_ANALYZE_FAILURE_THRESHOLD = 2; +const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']); export interface AutoSyncProjectConfig { localPath: string; - gitnexusGroup?: string; + groupName?: string; branches: string[]; remoteUrls: string[]; } @@ -22,29 +26,16 @@ export interface AutoSyncProjectConfig { export interface AutoSyncConfig { configPath: string; syncIntervalMinutes: number; + repoGitTimeoutMs: number; + maxConcurrency: number; + analyzeFailureThreshold: number; projects: AutoSyncProjectConfig[]; } -export type AutoSyncFlagDecision = - | { enabled: true } - | { enabled: false; reason: 'unset' | 'disabled' | 'invalid'; message?: string }; - export type AutoSyncConfigLoadResult = | { ok: true; config: AutoSyncConfig } | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; -export function parseAutoSyncFlag(raw = process.env[AUTO_SYNC_FLAG]): AutoSyncFlagDecision { - if (raw === undefined || raw.trim() === '') return { enabled: false, reason: 'unset' }; - const trimmed = raw.trim(); - if (trimmed === '0') return { enabled: false, reason: 'disabled' }; - if (trimmed === '1') return { enabled: true }; - return { - enabled: false, - reason: 'invalid', - message: `[auto-sync] ${AUTO_SYNC_FLAG} must be 0 or 1; got "${trimmed}". Auto sync is disabled.`, - }; -} - export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); } @@ -92,7 +83,7 @@ export async function loadAutoSyncConfig( return { ok: false, reason: 'invalid', - message: `[auto-sync] Invalid sync_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + message: `[auto-sync] Invalid watch_config.yml: ${(err as Error).message}. Auto sync is skipped.`, }; } } @@ -111,6 +102,31 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); } + const maxConcurrency = + raw.max_concurrency === undefined ? DEFAULT_MAX_CONCURRENCY : Number(raw.max_concurrency); + if (!Number.isInteger(maxConcurrency) || maxConcurrency <= 0) { + errors.push('max_concurrency must be a positive integer'); + } + + const repoGitTimeoutMs = + raw.repo_git_timeout === undefined + ? DEFAULT_REPO_GIT_TIMEOUT_MS + : parseDurationMs(raw.repo_git_timeout); + if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) { + errors.push('repo_git_timeout must be a positive duration such as 10s'); + } + + const analyzeFailureThreshold = + raw.analyze_failure_threshold === undefined + ? DEFAULT_ANALYZE_FAILURE_THRESHOLD + : Number(raw.analyze_failure_threshold); + if ( + !Number.isInteger(analyzeFailureThreshold) || + analyzeFailureThreshold < MIN_ANALYZE_FAILURE_THRESHOLD + ) { + errors.push(`analyze_failure_threshold must be an integer >= ${MIN_ANALYZE_FAILURE_THRESHOLD}`); + } + const rawProjects = raw.projects; if (!Array.isArray(rawProjects) || rawProjects.length === 0) { errors.push('projects must contain at least one project'); @@ -142,22 +158,86 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy if (remoteUrls.length === 0) { errors.push(`projects[${index}].remote_urls must contain at least one URL`); } + for (let urlIndex = 0; urlIndex < remoteUrls.length; urlIndex += 1) { + try { + validateAutoSyncRemoteUrl(remoteUrls[urlIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].remote_urls[${urlIndex}] ${(err as Error).message}`); + } + } - const branches = parseBranchCandidates(project.branch); - if (branches.length === 0) errors.push(`projects[${index}].branch is required`); + if (project.branch !== undefined && project.branches !== undefined) { + errors.push(`projects[${index}] must not set both branch and branches`); + } + const branches = parseBranchCandidates( + project.branches !== undefined ? project.branches : project.branch, + ); + if (branches.length === 0) errors.push(`projects[${index}].branches is required`); + for (let branchIndex = 0; branchIndex < branches.length; branchIndex += 1) { + try { + validateAutoSyncBranchName(branches[branchIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].branches[${branchIndex}] ${(err as Error).message}`); + } + } - const gitnexusGroup = - typeof project.gitnexus_group === 'string' ? project.gitnexus_group.trim() : undefined; - if (gitnexusGroup && !GROUP_NAME_PATTERN.test(gitnexusGroup)) { - errors.push(`projects[${index}].gitnexus_group is invalid`); + const groupName = + typeof project.group_name === 'string' && project.group_name.trim() + ? project.group_name.trim() + : undefined; + if (groupName && !GROUP_NAME_PATTERN.test(groupName)) { + errors.push(`projects[${index}].group_name is invalid`); } if (localPath && remoteUrls.length > 0 && branches.length > 0) { - projects.push({ localPath, gitnexusGroup, branches, remoteUrls }); + projects.push({ localPath, groupName, branches, remoteUrls }); } }); } if (errors.length > 0) throw new Error(errors.join('; ')); - return { configPath, syncIntervalMinutes: interval, projects }; + return { + configPath, + syncIntervalMinutes: interval, + repoGitTimeoutMs, + maxConcurrency, + analyzeFailureThreshold, + projects, + }; +} + +export function validateAutoSyncRemoteUrl(remoteUrl: string): void { + const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim()); + if (!match) { + throw new Error('must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git'); + } + const host = match[1].toLowerCase(); + const repoPath = match[2]; + if (!ALLOWED_REMOTE_HOSTS.has(host)) { + throw new Error('host must be one of github.com, gitlab.com, or gitee.com'); + } + if (repoPath.startsWith('/') || repoPath.includes('..') || repoPath.split('/').length < 2) { + throw new Error('path must include owner/repo without traversal'); + } +} + +export function validateAutoSyncBranchName(branch: string): void { + if (!branch.trim()) throw new Error('must not be empty'); + if (/[\s\0-\x1f\x7f]/.test(branch)) throw new Error('must not contain whitespace or control characters'); + if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref'); + if (branch.startsWith('-')) throw new Error('must not start with "-"'); + if (branch.includes('..')) throw new Error('must not contain ".."'); + if (branch.includes('`')) throw new Error('must not contain backticks'); +} + +export function parseDurationMs(value: unknown): number { + if (typeof value === 'number') return value * 1_000; + const raw = String(value ?? '').trim(); + const match = /^(\d+)(ms|s|m)?$/.exec(raw); + if (!match) return Number.NaN; + const amount = Number(match[1]); + const unit = match[2] ?? 's'; + if (unit === 'ms') return amount; + if (unit === 's') return amount * 1_000; + return amount * 60_000; } diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts index de1c0b4f7..6dc23a177 100644 --- a/gitnexus/src/core/auto-sync/index.ts +++ b/gitnexus/src/core/auto-sync/index.ts @@ -1,25 +1,29 @@ export { AUTO_SYNC_CONFIG_FILE, - AUTO_SYNC_FLAG, getAutoSyncConfigPath, loadAutoSyncConfig, parseAutoSyncConfig, - parseAutoSyncFlag, parseBranchCandidates, + parseDurationMs, + validateAutoSyncBranchName, + validateAutoSyncRemoteUrl, type AutoSyncConfig, type AutoSyncConfigLoadResult, - type AutoSyncFlagDecision, type AutoSyncProjectConfig, } from './config.js'; export { buildStateKey, + getAutoSyncWatchDir, getAutoSyncStatePath, + getProjectCommitInfoPath, loadAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, + writeProjectCommitInfo, type AutoSyncAnalyzeStatus, type AutoSyncCommitState, type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, } from './state.js'; export { extractRepoNameFromRemoteUrl } from './repo.js'; export { @@ -31,10 +35,19 @@ export { export { addRepoToGroup, getConfiguredRepoPath, + resolveActualConcurrency, runAutoSyncOnce, syncGroupByName, type AutoSyncLogger, type AutoSyncRunDeps, type AutoSyncRunResult, } from './runner.js'; -export { maybeStartAutoSyncFromEnv, type AutoSyncStartHandle } from './starter.js'; +export { + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + startAutoSyncWatch, + stopAutoSyncWatch, + type AutoSyncStartHandle, + type AutoSyncWatchPaths, + type WatchStatusRecord, +} from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts index 10f5d6ab7..e8510c248 100644 --- a/gitnexus/src/core/auto-sync/path-security.ts +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; +import { getAutoSyncWatchDir } from './state.js'; const DANGEROUS_ROOTS = new Set( [ @@ -61,6 +62,7 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise number; } export interface AutoSyncRunResult { @@ -59,9 +65,11 @@ const DEFAULT_DEPS: AutoSyncRunDeps = { registerRepo, loadState: loadAutoSyncState, saveState: saveAutoSyncState, + writeCommitInfo: writeProjectCommitInfo, addRepoToGroup, syncGroupByName, resolveCloneRoot: resolveConfiguredCloneRoot, + getAvailableMemoryGB: () => Math.floor(process.availableMemory?.() ?? 0) / 1024 / 1024 / 1024, }; export async function runAutoSyncOnce( @@ -74,89 +82,192 @@ export async function runAutoSyncOnce( const state = await deps.loadState(); const groupsToSync = new Set(); const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + const commitInfoEntries: ProjectCommitInfoEntry[] = []; + const actualConcurrency = resolveActualConcurrency(config.maxConcurrency, deps.getAvailableMemoryGB()); + logger.info( + `[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`, + ); - for (const project of config.projects) { - for (const remoteUrl of project.remoteUrls) { - try { - const repoName = extractRepoNameFromRemoteUrl(remoteUrl); - const cloneRoot = await deps.resolveCloneRoot(project.localPath); - const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); - await deps.cloneOrPull(remoteUrl, targetDir, undefined, { - allowedCloneRoot: cloneRoot.root, - expectedRepoName: repoName, - quarantineRoot: cloneRoot.quarantineRoot, - }); - result.synced += 1; - - const currentBranch = deps.getCurrentBranch(targetDir); - if (!currentBranch || !project.branches.includes(currentBranch)) { - result.skippedAnalysis += 1; - logger.warn( - `[auto-sync] Skip analysis for ${targetDir}; current branch ${currentBranch ?? ''} is not in configured branches: ${project.branches.join(', ')}.`, - ); - continue; - } - - const branch = currentBranch; - const currentCommit = deps.getCurrentCommit(targetDir); - const stateKey = buildStateKey(targetDir, branch); - const previous = state[stateKey]; - let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; - let analyzedCommitId = previous?.analyzedCommitId; - - if ( - shouldAnalyzeCommit({ - currentCommit, - previousAnalyzedCommit: previous?.analyzedCommitId, - previousStatus: previous?.lastAnalyzeStatus, - }) - ) { - try { - const analysis = await deps.runFullAnalysis( - targetDir, - { branch, skipAgentsMd: true, skipSkills: true }, - { onProgress: () => {} }, - ); - const meta: RepoMeta = { - repoPath: targetDir, - lastCommit: currentCommit, - indexedAt: now().toISOString(), - stats: analysis.stats, - branch, - }; - await deps.registerRepo(targetDir, meta, { name: repoName, allowDuplicateName: true }); - analyzeStatus = 'success'; - analyzedCommitId = currentCommit; - result.analyzed += 1; - } catch (err: unknown) { - analyzeStatus = 'failed'; - result.failed += 1; - logger.error(`[auto-sync] Analysis failed for ${targetDir}: ${(err as Error).message}`); - } - } else { - result.skippedAnalysis += 1; - logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); - } - - state[stateKey] = { - codeCommitId: currentCommit, - analyzedCommitId, - lastAnalyzeStatus: analyzeStatus, - lastSyncTime: now().toISOString(), + const workItems = await buildWorkItems(config, deps); + const repoResults = await mapWithConcurrency(workItems, actualConcurrency, async (item) => { + const lastSyncTime = now().toISOString(); + try { + validateAutoSyncRemoteUrl(item.remoteUrl); + const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName); + const syncResult = await syncFirstAvailableBranch({ + item, + repoName, + targetDir, + timeoutMs: config.repoGitTimeoutMs, + deps, + logger, + }); + if (syncResult.ok === false) { + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir, + branch: item.project.branches[0], + status: syncResult.status, + analyzeConsecutiveFailures: 0, + lastSyncTime, }; + } - if (project.gitnexusGroup) { - const added = await deps.addRepoToGroup(project, repoName); - if (added) groupsToSync.add(project.gitnexusGroup); + const currentBranch = syncResult.branch; + + const currentCommit = deps.getCurrentCommit(targetDir); + const stateKey = buildStateKey(targetDir, currentBranch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; + let lastAnalyzeError = previous?.lastAnalyzeError; + let stats: RepoMeta['stats'] | undefined; + + if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { + analyzeStatus = 'threshold_skipped'; + logger.error( + `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, + ); + } else if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runFullAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + stats = analysis.stats; + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } catch (err: unknown) { + analyzeStatus = 'failed'; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = shortErrorMessage(err); + logger.error( + `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, + ); } + } else { + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + + return { + kind: 'synced' as const, + project: item.project, + repoName, + remoteUrl: item.remoteUrl, + targetDir, + branch: currentBranch, + currentCommit, + analyzedCommitId, + analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + stats, + stateKey, + lastSyncTime, + }; + } catch (err: unknown) { + logger.error(`[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir: '', + status: 'sync_failed' as const, + lastSyncTime, + }; + } + }); + + for (const repoResult of repoResults) { + if (repoResult.kind === 'failed') { + result.failed += 1; + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + status: repoResult.status, + lastSyncTime: repoResult.lastSyncTime, + }); + continue; + } + + result.synced += 1; + const stateEntry: AutoSyncCommitStateEntry = { + codeCommitId: repoResult.currentCommit, + analyzedCommitId: repoResult.analyzedCommitId, + lastAnalyzeStatus: repoResult.analyzeStatus, + analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, + lastAnalyzeError: repoResult.lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }; + state[repoResult.stateKey] = stateEntry; + if (repoResult.analyzeStatus === 'success') { + const meta: RepoMeta = { + repoPath: repoResult.targetDir, + lastCommit: repoResult.currentCommit, + indexedAt: repoResult.lastSyncTime, + stats: repoResult.stats!, + branch: repoResult.branch, + }; + await deps.registerRepo(repoResult.targetDir, meta, { + name: repoResult.repoName, + allowDuplicateName: true, + }); + result.analyzed += 1; + } else if (repoResult.analyzeStatus === 'failed') { + result.failed += 1; + } else if (repoResult.analyzeStatus === 'threshold_skipped') { + result.skippedAnalysis += 1; + } else { + result.skippedAnalysis += 1; + } + + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + codeCommitId: repoResult.currentCommit, + analyzedCommitId: repoResult.analyzedCommitId, + status: repoResult.analyzeStatus, + analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, + analyzeFailureThreshold: config.analyzeFailureThreshold, + lastAnalyzeError: repoResult.lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }); + + if (repoResult.project.groupName) { + let groupMembershipOk = false; + try { + await deps.addRepoToGroup(repoResult.project, repoResult.repoName); + groupMembershipOk = true; } catch (err: unknown) { result.failed += 1; - logger.error(`[auto-sync] Repository sync failed for ${remoteUrl}: ${(err as Error).message}`); + logger.error(`[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`); + } + if (groupMembershipOk && repoResult.analyzeStatus === 'success') { + groupsToSync.add(repoResult.project.groupName); } } } await deps.saveState(state); + await deps.writeCommitInfo(commitInfoEntries); for (const groupName of groupsToSync) { try { await deps.syncGroupByName(groupName); @@ -168,6 +279,11 @@ export async function runAutoSyncOnce( return result; } +function shortErrorMessage(err: unknown): string { + const message = (err as Error).message || String(err); + return message.replace(/\s+/g, ' ').slice(0, 240); +} + export function getConfiguredRepoPath( project: Pick, repoName: string, @@ -176,11 +292,11 @@ export function getConfiguredRepoPath( } export async function addRepoToGroup( - project: Pick, + project: Pick, repoName: string, ): Promise { - if (!project.gitnexusGroup) return false; - const groupDir = getGroupDir(getDefaultGitnexusDir(), project.gitnexusGroup); + if (!project.groupName) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName); const config = await loadGroupConfig(groupDir); if (Object.values(config.repos).includes(repoName)) return false; config.repos[repoName] = repoName; @@ -199,3 +315,100 @@ async function writeGroupConfigAtomic(filePath: string, config: unknown): Promis await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); await fs.rename(tmpPath, filePath); } + +export function resolveActualConcurrency(configured: number, availableMemoryGB: number): number { + const memoryLimit = Math.max(1, Math.floor(availableMemoryGB / 2)); + return Math.max(1, Math.min(configured, memoryLimit)); +} + +async function buildWorkItems(config: AutoSyncConfig, deps: AutoSyncRunDeps): Promise { + const items: AutoSyncWorkItem[] = []; + const targetOwners = new Map(); + for (const project of config.projects) { + const cloneRoot = await deps.resolveCloneRoot(project.localPath); + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); + const previous = targetOwners.get(targetDir); + if (previous !== undefined) { + throw new Error(`Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`); + } + targetOwners.set(targetDir, remoteUrl); + } catch (err: unknown) { + if ((err as Error).message.startsWith('Duplicate auto-sync targetDir')) throw err; + } + items.push({ project, remoteUrl, cloneRoot }); + } + } + return items; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let nextIndex = 0; + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (nextIndex < items.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await worker(items[currentIndex]); + } + }); + await Promise.all(runners); + return results; +} + +interface AutoSyncWorkItem { + project: AutoSyncProjectConfig; + remoteUrl: string; + cloneRoot: Awaited>; +} + +async function syncFirstAvailableBranch(input: { + item: AutoSyncWorkItem; + repoName: string; + targetDir: string; + timeoutMs: number; + deps: AutoSyncRunDeps; + logger: AutoSyncLogger; +}): Promise< + | { ok: true; branch: string } + | { ok: false; status: 'branch_unavailable' | 'sync_timeout'; message: string } +> { + const failures: string[] = []; + let sawTimeout = false; + for (const branch of input.item.project.branches) { + try { + await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, { + allowedCloneRoot: input.item.cloneRoot.root, + expectedRepoName: input.repoName, + quarantineRoot: input.item.cloneRoot.quarantineRoot, + allowAutoSyncSsh: true, + timeoutMs: input.timeoutMs, + branch, + }); + const currentBranch = input.deps.getCurrentBranch(input.targetDir); + if (currentBranch === branch) return { ok: true, branch }; + failures.push(`${branch}: checked out ${currentBranch ?? ''}`); + input.logger.warn( + `[auto-sync] Branch ${branch} for ${input.item.remoteUrl} synced but current branch is ${currentBranch ?? ''}; trying next branch.`, + ); + } catch (err: unknown) { + const message = (err as Error).message; + if (message.includes('timed out')) sawTimeout = true; + failures.push(`${branch}: ${message}`); + input.logger.warn( + `[auto-sync] Branch ${branch} unavailable for ${input.item.remoteUrl}: ${message}`, + ); + } + } + return { + ok: false, + status: sawTimeout ? 'sync_timeout' : 'branch_unavailable', + message: failures.join('; '), + }; +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index 579b18ab5..8d4b00a16 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -1,28 +1,95 @@ -import { loadAutoSyncConfig, parseAutoSyncFlag } from './config.js'; +import fs from 'node:fs/promises'; +import crypto from 'node:crypto'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { loadAutoSyncConfig } from './config.js'; import { runAutoSyncOnce } from './runner.js'; +import { getAutoSyncWatchDir } from './state.js'; export interface AutoSyncStartHandle { - stop(): void; + stop(): Promise; } -export async function maybeStartAutoSyncFromEnv(options: { +export type WatchStatusState = 'running' | 'stopping' | 'stopped' | 'stale' | 'error'; + +export interface WatchStatusRecord { + state: WatchStatusState; + pid?: number; + ownerId?: string; + configPath?: string; + message?: string; + updatedAt: string; +} + +export interface WatchLockRecord { + pid: number; + ownerId: string; + createdAt: string; +} + +export interface AutoSyncWatchPaths { + pidPath: string; + lockPath: string; + statusPath: string; +} + +export interface AutoSyncWatchControlDeps { + isProcessAlive(pid: number): boolean; + killProcess(pid: number, signal?: NodeJS.Signals): void; + sleep(ms: number): Promise; +} + +export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWatchPaths { + const watchDir = getAutoSyncWatchDir(gitnexusDir); + return { + pidPath: path.join(watchDir, 'watch.pid'), + lockPath: path.join(watchDir, 'watch.lock'), + statusPath: path.join(watchDir, 'watch.status.json'), + }; +} + +export async function startAutoSyncWatch(options: { setIntervalFn?: typeof setInterval; clearIntervalFn?: typeof clearInterval; runOnce?: typeof runAutoSyncOnce; stderr?: Pick; + keepAlive?: boolean; + paths?: AutoSyncWatchPaths; + deps?: Partial; } = {}): Promise { const stderr = options.stderr ?? process.stderr; - const flag = parseAutoSyncFlag(); - if (flag.enabled === false) { - if (flag.message) stderr.write(`${flag.message}\n`); - return null; - } + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const ownerId = crypto.randomUUID(); + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + const lockHandle = await acquireWatchLock(paths, deps, stderr); + if (!lockHandle) return null; + await lockHandle.writeFile( + `${JSON.stringify({ pid: process.pid, ownerId, createdAt: new Date().toISOString() })}\n`, + 'utf-8', + ); + await fs.writeFile(paths.pidPath, `${process.pid}\n`, 'utf-8'); const loaded = await loadAutoSyncConfig(); if (loaded.ok === false) { stderr.write(`${loaded.message}\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: process.pid, + ownerId, + message: loaded.message, + updatedAt: new Date().toISOString(), + }); + await cleanupWatchFiles(paths, lockHandle); return null; } + await writeWatchStatus(paths, { + state: 'running', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); const runOnce = options.runOnce ?? runAutoSyncOnce; let running = false; @@ -32,9 +99,17 @@ export async function maybeStartAutoSyncFromEnv(options: { return; } running = true; + const startedAt = new Date(); + stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); void runOnce(loaded.config) + .then((result) => { + stderr.write( + `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, + ); + }) .catch((err: unknown) => { stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + stderr.write('[auto-sync] Watch loop finished: failed.\n'); }) .finally(() => { running = false; @@ -46,6 +121,359 @@ export async function maybeStartAutoSyncFromEnv(options: { const setIntervalFn = options.setIntervalFn ?? setInterval; const clearIntervalFn = options.clearIntervalFn ?? clearInterval; const timer = setIntervalFn(runSafely, intervalMs); - timer.unref?.(); - return { stop: () => clearIntervalFn(timer) }; + if (options.keepAlive === false) timer.unref?.(); + return { + stop: async () => { + clearIntervalFn(timer); + await writeWatchStatus(paths, { + state: 'stopped', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }).finally(() => cleanupWatchFiles(paths, lockHandle)); + }, + }; +} + +async function acquireWatchLock( + paths: AutoSyncWatchPaths, + deps: AutoSyncWatchControlDeps, + stderr: Pick, +): Promise { + try { + return await fs.open(paths.lockPath, 'wx'); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + } + + const lock = await readLockFile(paths.lockPath); + if (!lock) { + const message = 'watch lock already exists but has no readable owner; refusing to start'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return null; + } + + if (deps.isProcessAlive(lock.pid)) { + stderr.write(`[auto-sync] Watch is already running with pid ${lock.pid}.\n`); + await writeWatchStatus(paths, { + state: 'running', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'watch already running', + updatedAt: new Date().toISOString(), + }); + return null; + } + + stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'removed stale lock and pid', + updatedAt: new Date().toISOString(), + }); + + try { + return await fs.open(paths.lockPath, 'wx'); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'EEXIST') { + const message = 'watch lock was reacquired by another process; refusing to start'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return null; + } + throw err; + } +} + +export async function stopAutoSyncWatch(options: { + paths?: AutoSyncWatchPaths; + stderr?: Pick; + deps?: Partial; + timeoutMs?: number; + pollMs?: number; +} = {}): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const timeoutMs = options.timeoutMs ?? 10_000; + const pollMs = options.pollMs ?? 100; + const pid = await readPid(paths.pidPath); + if (!pid) { + const lock = await readLockFile(paths.lockPath); + if (lock && deps.isProcessAlive(lock.pid)) { + const message = `watch appears to be starting with pid ${lock.pid}; pid file is not ready`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: lock.pid, + ownerId: lock.ownerId, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + if (lock) { + stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'removed stale lock without pid file', + updatedAt: new Date().toISOString(), + }); + return false; + } + if (await fileExists(paths.lockPath)) { + const message = 'watch lock exists but has no readable owner; refusing to stop'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + stderr.write('[auto-sync] Watch is not running.\n'); + await writeWatchStatus(paths, { + state: 'stopped', + message: 'no pid file', + updatedAt: new Date().toISOString(), + }); + return false; + } + if (!deps.isProcessAlive(pid)) { + stderr.write(`[auto-sync] Removing stale watch pid ${pid}.\n`); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid, + message: 'removed stale pid and lock', + updatedAt: new Date().toISOString(), + }); + return false; + } + const owner = await readVerifiedWatchOwner(paths, pid); + if (owner.ok === false) { + const message = `refusing to stop pid ${pid}; ${owner.reason}`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + pid, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + await writeWatchStatus(paths, { + state: 'stopping', + pid, + ownerId: owner.owner.ownerId, + message: 'stop signal sent; waiting for watch process to exit', + updatedAt: new Date().toISOString(), + }); + deps.killProcess(pid, 'SIGTERM'); + stderr.write(`[auto-sync] Stop signal sent to watch pid ${pid}.\n`); + const stopped = await waitForProcessExit(pid, { deps, timeoutMs, pollMs }); + if (!stopped) { + const message = `watch pid ${pid} did not exit within ${timeoutMs}ms`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'stopping', + pid, + ownerId: owner.owner.ownerId, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stopped', + pid, + ownerId: owner.owner.ownerId, + message: 'watch stopped', + updatedAt: new Date().toISOString(), + }); + return true; +} + +export async function readAutoSyncWatchStatus( + paths = getAutoSyncWatchPaths(), + deps: Partial = {}, +): Promise { + const resolvedDeps = resolveWatchDeps(deps); + const pid = await readPid(paths.pidPath); + if (pid && !resolvedDeps.isProcessAlive(pid)) { + return { + state: 'stale', + pid, + message: 'pid file exists but process is not running', + updatedAt: new Date().toISOString(), + }; + } + if (pid) { + const stored = await readStatusFile(paths.statusPath); + const owner = await readVerifiedWatchOwner(paths, pid); + if (owner.ok === false) { + return { + ...stored, + state: 'error', + pid, + message: owner.reason, + updatedAt: new Date().toISOString(), + }; + } + return { + ...stored, + state: stored?.state === 'stopping' ? 'stopping' : 'running', + pid, + ownerId: owner.owner.ownerId, + updatedAt: new Date().toISOString(), + }; + } + const stored = await readStatusFile(paths.statusPath); + return stored ?? { state: 'stopped', updatedAt: new Date().toISOString() }; +} + +async function readLockFile(lockPath: string): Promise { + try { + const raw = await fs.readFile(lockPath, 'utf-8'); + const parsed = JSON.parse(raw) as WatchLockRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId + ) { + return parsed; + } + return undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return undefined; + } +} + +async function readVerifiedWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, +): Promise<{ ok: true; owner: WatchLockRecord } | { ok: false; reason: string }> { + const [status, lock] = await Promise.all([ + readStatusFile(paths.statusPath), + readLockFile(paths.lockPath), + ]); + if (!lock) return { ok: false, reason: 'watch lock is missing or invalid' }; + if (!status) return { ok: false, reason: 'watch status is missing or invalid' }; + if (lock.pid !== pid) return { ok: false, reason: 'watch lock pid does not match pid file' }; + if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' }; + if (!status.ownerId || status.ownerId !== lock.ownerId) { + return { ok: false, reason: 'watch status owner does not match lock owner' }; + } + return { ok: true, owner: lock }; +} + +async function waitForProcessExit( + pid: number, + options: { deps: AutoSyncWatchControlDeps; timeoutMs: number; pollMs: number }, +): Promise { + const deadline = Date.now() + options.timeoutMs; + while (Date.now() < deadline) { + if (!options.deps.isProcessAlive(pid)) return true; + await options.deps.sleep(options.pollMs); + } + return !options.deps.isProcessAlive(pid); +} + +async function readPid(pidPath: string): Promise { + try { + const raw = await fs.readFile(pidPath, 'utf-8'); + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 ? pid : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw err; + } +} + +async function readStatusFile(statusPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(statusPath, 'utf-8')) as WatchStatusRecord; + return parsed && typeof parsed === 'object' ? parsed : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return { + state: 'error', + message: `unable to read status file: ${(err as Error).message}`, + updatedAt: new Date().toISOString(), + }; + } +} + +async function writeWatchStatus(paths: AutoSyncWatchPaths, record: WatchStatusRecord): Promise { + await fs.mkdir(path.dirname(paths.statusPath), { recursive: true }); + const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, paths.statusPath); +} + +async function cleanupWatchFiles(paths: AutoSyncWatchPaths, lockHandle?: fs.FileHandle): Promise { + await lockHandle?.close().catch(() => {}); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); +} + +async function removeIfExists(filePath: string): Promise { + await fs.rm(filePath, { force: true }); +} + +async function fileExists(filePath: string): Promise { + return fs.access(filePath).then( + () => true, + () => false, + ); +} + +function resolveWatchDeps(deps: Partial = {}): AutoSyncWatchControlDeps { + return { + isProcessAlive: + deps.isProcessAlive ?? + ((pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }), + killProcess: + deps.killProcess ?? + ((pid, signal = 'SIGTERM') => { + process.kill(pid, signal); + }), + sleep: + deps.sleep ?? + ((ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })), + }; } diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts index f14d9aa5f..0de053ca9 100644 --- a/gitnexus/src/core/auto-sync/state.ts +++ b/gitnexus/src/core/auto-sync/state.ts @@ -2,19 +2,29 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; -export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped'; +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped'; export interface AutoSyncCommitStateEntry { codeCommitId: string; analyzedCommitId?: string; lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + analyzeConsecutiveFailures?: number; + lastAnalyzeError?: string; lastSyncTime: string; } export type AutoSyncCommitState = Record; +export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'watch'); +} + export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { - return path.join(gitnexusDir, 'auto-sync-state.json'); + return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json'); +} + +export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt'); } export function buildStateKey(repoPath: string, branch: string): string { @@ -59,3 +69,46 @@ export async function saveAutoSyncState( await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); await fs.rename(tmpPath, statePath); } + +export async function writeProjectCommitInfo( + entries: ProjectCommitInfoEntry[], + infoPath = getProjectCommitInfoPath(), +): Promise { + await fs.mkdir(path.dirname(infoPath), { recursive: true }); + const lines = [ + '# GitNexus auto-sync project commit info', + `updated_at: ${new Date().toISOString()}`, + '', + ...entries.flatMap((entry) => [ + `remote: ${entry.remoteUrl}`, + `local_path: ${entry.localPath}`, + `branch: ${entry.branch ?? ''}`, + `code_commit: ${entry.codeCommitId ?? ''}`, + `analyzed_commit: ${entry.analyzedCommitId ?? ''}`, + `status: ${entry.status}`, + `analyze_consecutive_failures: ${entry.analyzeConsecutiveFailures ?? 0}`, + ...(entry.analyzeFailureThreshold === undefined + ? [] + : [`analyze_failure_threshold: ${entry.analyzeFailureThreshold}`]), + ...(entry.lastAnalyzeError ? [`last_analyze_error: ${entry.lastAnalyzeError}`] : []), + `last_sync_time: ${entry.lastSyncTime}`, + '', + ]), + ]; + const tmpPath = `${infoPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${lines.join('\n')}\n`, 'utf-8'); + await fs.rename(tmpPath, infoPath); +} + +export interface ProjectCommitInfoEntry { + remoteUrl: string; + localPath: string; + branch?: string; + codeCommitId?: string; + analyzedCommitId?: string; + status: AutoSyncAnalyzeStatus | 'sync_failed' | 'branch_skipped' | 'branch_unavailable' | 'sync_timeout'; + analyzeConsecutiveFailures?: number; + analyzeFailureThreshold?: number; + lastAnalyzeError?: string; + lastSyncTime: string; +} diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index bb4cfcef4..93670aa0d 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -34,7 +34,7 @@ import { fileURLToPath } from 'url'; import { JobManager } from './analyze-job.js'; import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js'; import { - extractRepoName, + extractWebRepoName, getCloneDir, cloneOrPull, warnIfInsecureAzureConfig, @@ -1582,7 +1582,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => try { // Clone if URL provided if (repoUrl && !repoLocalPath) { - const repoName = extractRepoName(repoUrl); + const repoName = extractWebRepoName(repoUrl); targetPath = getCloneDir(repoName); jobManager.updateJob(job.id, { diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 588f36ea3..60f0b44d6 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -11,10 +11,12 @@ import fs from 'fs/promises'; import { isIP } from 'net'; import { logger } from '../core/logger.js'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { sanitizeRepoName } from '../storage/git.js'; import { assertDirectoryOwnerAndPermissions, quarantineAutoSyncPartial, } from '../core/auto-sync/path-security.js'; +import { validateAutoSyncRemoteUrl } from '../core/auto-sync/config.js'; /** * Root directory for all cloned repositories. Targets must resolve inside this. @@ -59,6 +61,26 @@ export function extractRepoName(url: string): string { return name; } +/** + * Derive a clone directory name for the web `/api/analyze` boundary. + * + * The API historically accepted Azure DevOps and similar URLs whose repo + * segment contains spaces or other directory-unsafe characters by sanitizing + * the final segment. Keep that compatibility at the web boundary while leaving + * `extractRepoName()` strict for internal/security-sensitive callers. + */ +export function extractWebRepoName(url: string): string { + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const rawName = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; + const safeName = sanitizeRepoName(rawName); + if (!rawName || safeName === 'unknown') { + throw new Error('Could not extract a valid repository name from URL'); + } + return safeName; +} + /** Get the clone target directory for a repo name. */ export function getCloneDir(repoName: string): string { // Re-validate at the boundary even though extractRepoName already checked — @@ -247,9 +269,20 @@ export interface CloneOrPullOptions { allowedCloneRoot?: string; expectedRepoName?: string; quarantineRoot?: string; + allowAutoSyncSsh?: boolean; + timeoutMs?: number; + branch?: string; runGitForTest?: typeof runGit; } +type RunGitOptions = { + token?: string; + url?: string; + timeoutMs?: number; + timeoutKillGraceMs?: number; + spawnForTest?: typeof spawn; +}; + /** * Build the `git clone` argument list for a given URL and target directory. * @@ -319,6 +352,10 @@ export function buildCloneArgs(url: string, targetDir: string): string[] { return ['clone', '--depth', '1', '--', url, targetDir]; } +export function buildBranchCloneArgs(url: string, targetDir: string, branch: string): string[] { + return ['clone', '--depth', '1', '--branch', branch, '--', url, targetDir]; +} + /** * Normalize a git URL into a comparable form. * @@ -486,7 +523,8 @@ export async function cloneOrPull( // Always validate the requested URL — the prior shape only ran this in // the code path where the repo was cloned. Now it runs unconditionally, // preventing SSRF / blocked-host bypasses even when targetDir already exists. - validateGitUrl(url); + if (options?.allowAutoSyncSsh) validateAutoSyncRemoteUrl(url); + else validateGitUrl(url); await fs.mkdir(cloneRoot, { recursive: true }); if (options?.allowedCloneRoot) { await assertDirectoryOwnerAndPermissions(cloneRoot); @@ -511,7 +549,24 @@ export async function cloneOrPull( // whatever remote the dir was originally cloned from. await assertRemoteMatchesRequestedUrl(safeTarget, url); onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); - await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url }); + const runGitImpl = options?.runGitForTest ?? runGit; + if (options?.branch) { + await runGitImpl(['fetch', '--depth', '1', 'origin', options.branch], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + await runGitImpl(['checkout', options.branch], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + } + await runGitImpl(['pull', '--ff-only'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); } else { if (targetExists) { throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); @@ -522,7 +577,14 @@ export async function cloneOrPull( onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); try { const runGitImpl = options?.runGitForTest ?? runGit; - await runGitImpl(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + const cloneArgs = options?.branch + ? buildBranchCloneArgs(url, safeTarget, options.branch) + : buildCloneArgs(url, safeTarget); + await runGitImpl(cloneArgs, undefined, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); await assertPostRealpathContainment(cloneRoot, safeTarget); } catch (err: unknown) { if (options?.quarantineRoot) { @@ -731,10 +793,11 @@ export function buildGitEnv( function runGit( args: string[], cwd?: string, - options?: { token?: string; url?: string }, + options?: RunGitOptions, ): Promise { return new Promise((resolve, reject) => { - const proc = spawn('git', args, { + const spawnGit = options?.spawnForTest ?? spawn; + const proc = spawnGit('git', args, { cwd, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, @@ -742,21 +805,47 @@ function runGit( }); let stderr = ''; + let settled = false; + let timedOut = false; + let killTimer: NodeJS.Timeout | undefined; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + fn(); + }; + const timer = + options?.timeoutMs && options.timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + proc.kill('SIGTERM'); + killTimer = setTimeout(() => { + proc.kill('SIGKILL'); + }, options.timeoutKillGraceMs ?? 1_000); + }, options.timeoutMs) + : undefined; proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk; }); proc.on('close', (code) => { - if (code === 0) resolve(); + if (timedOut) { + finish(() => reject(new Error(`git ${args[0]} timed out after ${options?.timeoutMs}ms`))); + return; + } + if (code === 0) finish(resolve); else { // Log full stderr internally but don't expose it to API callers (SSRF mitigation) if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`); - reject(new Error(`git ${args[0]} failed (exit code ${code})`)); + finish(() => reject(new Error(`git ${args[0]} failed (exit code ${code})`))); } }); proc.on('error', (err) => { - reject(new Error(`Failed to spawn git: ${err.message}`)); + finish(() => reject(new Error(`Failed to spawn git: ${err.message}`))); }); }); } + +export const runGitForTest = runGit; diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 59e358c6a..9e8c5b13e 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -6,18 +6,25 @@ import { describe, expect, it, vi } from 'vitest'; import { addRepoToGroup, getConfiguredRepoPath, - maybeStartAutoSyncFromEnv, + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + resolveActualConcurrency, runAutoSyncOnce, + startAutoSyncWatch, + stopAutoSyncWatch, } from '../../src/core/auto-sync/index.js'; -import type { AutoSyncConfig, AutoSyncRunDeps } from '../../src/core/auto-sync/index.js'; +import type { AutoSyncConfig, AutoSyncRunDeps, AutoSyncWatchPaths } from '../../src/core/auto-sync/index.js'; const config: AutoSyncConfig = { - configPath: '/tmp/.gitnexus/sync_config.yml', + configPath: '/tmp/.gitnexus/watch_config.yml', syncIntervalMinutes: 10, + repoGitTimeoutMs: 10_000, + maxConcurrency: 1, + analyzeFailureThreshold: 3, projects: [ { localPath: '/tmp/repos', - gitnexusGroup: 'back_end', + groupName: 'back_end', branches: ['master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }, @@ -26,7 +33,7 @@ const config: AutoSyncConfig = { const cloneRoot = { root: '/tmp/repos', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', quarantineRetentionDays: 14, }; @@ -37,6 +44,25 @@ function withCloneRoot(deps: Partial): Partial }; } +async function writeWatchOwner(paths: AutoSyncWatchPaths, pid: number, ownerId = `owner-${pid}`): Promise { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile(paths.pidPath, `${pid}\n`); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid, ownerId, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ + state: 'running', + pid, + ownerId, + updatedAt: '2026-06-30T00:00:00.000Z', + })}\n`, + ); + return ownerId; +} + describe('auto-sync runner', () => { it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { const deps: Partial = withCloneRoot({ @@ -54,8 +80,10 @@ describe('auto-sync runner', () => { }, })), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -72,7 +100,10 @@ describe('auto-sync runner', () => { { allowedCloneRoot: '/tmp/repos', expectedRepoName: 'qts_account', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', }, ); expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); @@ -87,6 +118,70 @@ describe('auto-sync runner', () => { { name: 'qts_account', allowDuplicateName: true }, ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + status: 'success', + }), + ]); + }); + + it('syncs a group when a repo is newly added to the group', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('syncs a group after successful re-analysis even when membership already exists', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-3'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 2 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(1); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); }); it('skips analysis when commit id has not changed', async () => { @@ -105,8 +200,10 @@ describe('auto-sync runner', () => { }, })), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => false), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -131,8 +228,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => false), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); await runAutoSyncOnce(config, { @@ -147,12 +246,121 @@ describe('auto-sync runner', () => { { allowedCloneRoot: '/tmp/repos', expectedRepoName: 'qts_account', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', }, ); }); - it('skips analysis when the checked out branch is not configured', async () => { + it('falls back through configured branches and analyzes the first pullable branch', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_remoteUrl, _targetDir, _progress, options) => { + if (options?.branch === 'missing') throw new Error('remote branch not found'); + return '/tmp/repos/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 1, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + expect.objectContaining({ branch: 'missing' }), + ); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 2, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + expect.objectContaining({ branch: 'develop' }), + ); + expect(deps.runFullAnalysis).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + { branch: 'develop', skipAgentsMd: true, skipSkills: true }, + { onProgress: expect.any(Function) }, + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).not.toHaveBeenCalled(); + }); + + it('records branch_unavailable when all configured branches fail', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => { + throw new Error('remote branch not found'); + }), + getCurrentBranch: vi.fn(), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + branch: 'missing', + status: 'branch_unavailable', + }), + ]); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledTimes(2); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch develop unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).toHaveBeenCalledTimes(1); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Repository sync failed for git@gitee.com:qts_server/qts_account.git; no configured branch could be pulled: missing: remote branch not found; develop: remote branch not found', + ); + }); + + it('records branch_unavailable when checkout ends on an unexpected branch', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), @@ -162,8 +370,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -171,16 +381,16 @@ describe('auto-sync runner', () => { logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); expect(deps.runFullAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( - '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch develop is not in configured branches: master.', + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is develop; trying next branch.', ); }); - it('skips analysis when the checked out repository is detached', async () => { + it('records branch_unavailable when the checked out repository is detached', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), @@ -190,8 +400,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -199,16 +411,16 @@ describe('auto-sync runner', () => { logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); expect(deps.runFullAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( - '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch is not in configured branches: master.', + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is ; trying next branch.', ); }); - it('isolates repository, analysis, and group sync failures', async () => { + it('isolates repository and analysis failures without syncing groups for failed analysis', async () => { const errorLogger = vi.fn(); const failingConfig: AutoSyncConfig = { ...config, @@ -235,10 +447,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), - syncGroupByName: vi.fn(async () => { - throw new Error('group sync failed'); - }), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(failingConfig, { @@ -247,11 +459,11 @@ describe('auto-sync runner', () => { now: () => new Date('2026-06-30T00:00:00.000Z'), }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 3 }); + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 2 }); expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); expect(deps.registerRepo).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).toHaveBeenCalledWith(failingConfig.projects[0], 'qts_account'); - expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.syncGroupByName).not.toHaveBeenCalled(); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ '/tmp/repos/qts_account|master': expect.objectContaining({ @@ -266,11 +478,311 @@ describe('auto-sync runner', () => { expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), ); + }); + + it('reports group sync failures after successful analysis', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => { + throw new Error('group sync failed'); + }), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Group sync failed for back_end'), ); }); + it('caps actual concurrency by available memory and runs clone/analyze work concurrently', async () => { + const events: string[] = []; + let releaseFirstClone: (() => void) | undefined; + const concurrentConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 4, + projects: [ + { + ...config.projects[0], + groupName: undefined, + remoteUrls: ['git@github.com:owner/one.git', 'git@gitlab.com:owner/two.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + events.push(`clone-start:${remoteUrl}`); + if (remoteUrl.includes('/one.git')) { + await new Promise((resolve) => { + releaseFirstClone = resolve; + setTimeout(resolve, 0); + }); + } else { + releaseFirstClone?.(); + } + events.push(`clone-end:${remoteUrl}`); + return remoteUrl.includes('/one.git') ? '/tmp/repos/one' : '/tmp/repos/two'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn((repoPath) => (repoPath.endsWith('/one') ? 'one-commit' : 'two-commit')), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 4), + }); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await runAutoSyncOnce(concurrentConfig, { deps, logger }); + + expect(result.synced).toBe(2); + expect(logger.info).toHaveBeenCalledWith( + '[auto-sync] Starting sync loop with max_concurrency=2 analyze_failure_threshold=3.', + ); + expect(events.slice(0, 2)).toEqual([ + 'clone-start:git@github.com:owner/one.git', + 'clone-start:git@gitlab.com:owner/two.git', + ]); + expect(deps.registerRepo).toHaveBeenCalledTimes(2); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('rejects duplicate resolved targetDir before clone work starts', async () => { + const duplicateConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 2, + projects: [ + { + ...config.projects[0], + remoteUrls: ['git@github.com:owner/repo.git', 'git@gitlab.com:group/repo.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(duplicateConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).rejects.toThrow('Duplicate auto-sync targetDir'); + + expect(deps.cloneOrPull).not.toHaveBeenCalled(); + expect(deps.saveState).not.toHaveBeenCalled(); + expect(deps.writeCommitInfo).not.toHaveBeenCalled(); + }); + + it('rejects non auto-sync SSH URLs at runner boundary', async () => { + const invalidConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], remoteUrls: ['https://github.com/owner/repo.git'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(invalidConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.failed).toBe(1); + expect(deps.cloneOrPull).not.toHaveBeenCalled(); + }); + + it('increments analyze failure count and writes threshold details', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => { + throw new Error('parser crashed\nwith stack'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 1, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'parser crashed with stack', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'failed', + analyzeConsecutiveFailures: 2, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed with stack', + }), + ]); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Analysis failed for /tmp/repos/qts_account; consecutive failures 2/3: parser crashed with stack', + ); + }); + + it('skips analyze when consecutive failures have reached the threshold', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 3, + lastAnalyzeError: 'parser crashed', + lastAnalyzeStatus: 'threshold_skipped', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'threshold_skipped', + analyzeConsecutiveFailures: 3, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed', + }), + ]); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; analyze consecutive failures 3/3 reached threshold. Fix the repository or clear auto-sync state before retrying.', + ); + }); + + it('clears prior analyze failure count after a successful analyze', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result.analyzed).toBe(1); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 0, + lastAnalyzeError: undefined, + lastAnalyzeStatus: 'success', + }), + }), + ); + }); + + it('resolves actual concurrency from configured value and memory', () => { + expect(resolveActualConcurrency(8, 10)).toBe(5); + expect(resolveActualConcurrency(8, 1)).toBe(1); + expect(resolveActualConcurrency(2, 10)).toBe(2); + }); + it('detects existing groupPath to registryName mappings as already joined', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-group-')); @@ -288,7 +800,7 @@ describe('auto-sync runner', () => { ].join('\n'), ); - await expect(addRepoToGroup({ gitnexusGroup: 'back_end' }, 'qts_account')).resolves.toBe( + await expect(addRepoToGroup({ groupName: 'back_end' }, 'qts_account')).resolves.toBe( false, ); @@ -304,59 +816,54 @@ describe('auto-sync runner', () => { }); describe('auto-sync starter', () => { - it('does not read config or register timers when the flag is disabled', async () => { - const previous = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '0'; - const setIntervalFn = vi.fn() as unknown as typeof setInterval; - - try { - const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn }); - expect(handle).toBeNull(); - expect(setIntervalFn).not.toHaveBeenCalled(); - } finally { - if (previous === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previous; - } - }); - - it('registers a clearable timer when enabled with a valid config', async () => { - const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + it('registers a clearable timer with a valid fixed config', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); const timer = { unref: vi.fn() }; const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; const runOnce = vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); + const stderr = { write: vi.fn() }; try { - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; process.env.GITNEXUS_HOME = tempDir; await fs.writeFile( - path.join(tempDir, 'sync_config.yml'), + path.join(tempDir, 'watch_config.yml'), [ 'sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', - ' gitnexus_group: back_end', + ' group_name: back_end', ' branch: master', ' remote_urls:', ' - git@gitee.com:qts_server/qts_account.git', ].join('\n'), ); - const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn, clearIntervalFn, runOnce }); + const handle = await startAutoSyncWatch({ + setIntervalFn, + clearIntervalFn, + runOnce, + stderr, + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); expect(handle).not.toBeNull(); expect(runOnce).toHaveBeenCalledTimes(1); expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); expect(timer.unref).toHaveBeenCalled(); + await vi.waitFor(() => { + expect(stderr.write).toHaveBeenCalledWith(expect.stringContaining('[auto-sync] Watch loop started at ')); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch loop finished: synced=0 analyzed=0 skipped=0 failed=0.\n', + ); + }); - handle?.stop(); + await handle?.stop(); expect(clearIntervalFn).toHaveBeenCalledWith(timer); } finally { - if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; await fs.rm(tempDir, { recursive: true, force: true }); @@ -364,7 +871,6 @@ describe('auto-sync starter', () => { }); it('skips overlapping scheduled runs while a previous run is active', async () => { - const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); const timer = { unref: vi.fn() }; @@ -383,21 +889,20 @@ describe('auto-sync starter', () => { ); try { - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; process.env.GITNEXUS_HOME = tempDir; await fs.writeFile( - path.join(tempDir, 'sync_config.yml'), + path.join(tempDir, 'watch_config.yml'), [ 'sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', ' branch: master', ' remote_urls:', - ' - https://example.com/team/repo.git', + ' - git@github.com:team/repo.git', ].join('\n'), ); - await maybeStartAutoSyncFromEnv({ setIntervalFn, runOnce, stderr }); + await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); scheduled?.(); expect(runOnce).toHaveBeenCalledTimes(1); @@ -411,8 +916,334 @@ describe('auto-sync starter', () => { expect(runOnce).toHaveBeenCalledTimes(2); } finally { - if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses a second running watch for the same GITNEXUS_HOME', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + await writeWatchOwner(paths, 12345); + const handle = await startAutoSyncWatch({ + paths, + stderr, + deps: { isProcessAlive: vi.fn(() => true) }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('removes stale pid and lock before starting watch', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + try { + process.env.GITNEXUS_HOME = tempDir; + await writeWatchOwner(paths, 12345); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + setIntervalFn, + clearIntervalFn, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + + expect(handle).not.toBeNull(); + expect(await fs.readFile(paths.pidPath, 'utf-8')).toBe(`${process.pid}\n`); + await handle?.stop(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a half-initialized lock when pid has not been written yet', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + stderr, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + deps: { isProcessAlive: vi.fn(() => true) }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a live half-initialized lock when stop runs before pid is written', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + const killProcess = vi.fn(); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr, + deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + expect(killProcess).not.toHaveBeenCalled(); + expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); + expect(status).toMatchObject({ + state: 'error', + pid: 12345, + ownerId: 'starting-owner', + message: expect.stringContaining('appears to be starting'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('removes a stale half-initialized lock when stop runs before pid is written', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'stale-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { isProcessAlive: vi.fn(() => false), killProcess: vi.fn(), sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); + expect(status).toMatchObject({ + state: 'stale', + pid: 12345, + ownerId: 'stale-owner', + message: 'removed stale lock without pid file', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports status and sends stop signals from pid files', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + let alive = true; + try { + await writeWatchOwner(paths, 12345); + + await expect( + readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + ).resolves.toMatchObject({ state: 'running', pid: 12345 }); + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + pollMs: 1, + deps: { + isProcessAlive: vi.fn(() => alive), + killProcess: vi.fn((pid, signal) => { + killProcess(pid, signal); + alive = false; + }), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(true); + + expect(killProcess).toHaveBeenCalledWith(12345, 'SIGTERM'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + await expect(readAutoSyncWatchStatus(paths)).resolves.toMatchObject({ state: 'stopped' }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not mark stopped when stop times out waiting for the owner process', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + timeoutMs: 2, + pollMs: 1, + deps: { + isProcessAlive: vi.fn(() => true), + killProcess: vi.fn(), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(false); + + await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + state: 'stopping', + pid: 12345, + message: expect.stringContaining('did not exit'), + }); + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses to stop when pid status and lock ownership disagree', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile(paths.pidPath, '12345\n'); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'lock-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ state: 'running', pid: 12345, ownerId: 'other-owner', updatedAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + expect(killProcess).not.toHaveBeenCalled(); + await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('owner'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('restart can start only after stop confirms pid and lock cleanup', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + let alive = true; + try { + process.env.GITNEXUS_HOME = tempDir; + await writeWatchOwner(paths, 12345); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + await expect( + stopAutoSyncWatch({ + paths, + timeoutMs: 10, + pollMs: 1, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => alive), + killProcess: vi.fn(() => { + alive = false; + }), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(true); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + + const handle = await startAutoSyncWatch({ + paths, + setIntervalFn, + clearIntervalFn, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + expect(handle).not.toBeNull(); + await handle?.stop(); + } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; await fs.rm(tempDir, { recursive: true, force: true }); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 6cc636ba6..cc8be6e17 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -5,20 +5,25 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { extractRepoNameFromRemoteUrl, + getAutoSyncStatePath, + getAutoSyncWatchDir, + getProjectCommitInfoPath, loadAutoSyncConfig, - parseAutoSyncFlag, parseBranchCandidates, + parseDurationMs, resolveConfiguredCloneRoot, loadAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, + validateAutoSyncRemoteUrl, + validateAutoSyncBranchName, + writeProjectCommitInfo, } from '../../src/core/auto-sync/index.js'; describe('auto-sync', () => { let tempDir: string; let gitnexusHome: string; let oldHome: string | undefined; - let oldFlag: string | undefined; beforeEach(async () => { const base = path.join(process.cwd(), '.tmp-test'); @@ -27,43 +32,38 @@ describe('auto-sync', () => { gitnexusHome = path.join(tempDir, '.gitnexus'); await fs.mkdir(gitnexusHome); oldHome = process.env.GITNEXUS_HOME; - oldFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; process.env.GITNEXUS_HOME = gitnexusHome; - delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; }); afterEach(async () => { if (oldHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = oldHome; - if (oldFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = oldFlag; await fs.rm(tempDir, { recursive: true, force: true }); vi.restoreAllMocks(); }); - it('keeps auto sync disabled when the flag is unset or 0', () => { - expect(parseAutoSyncFlag(undefined)).toEqual({ enabled: false, reason: 'unset' }); - expect(parseAutoSyncFlag('0')).toEqual({ enabled: false, reason: 'disabled' }); + it('places watch runtime artifacts under the watch directory by default', () => { + expect(getAutoSyncWatchDir(gitnexusHome)).toBe(path.join(gitnexusHome, 'watch')); + expect(getAutoSyncStatePath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'auto-sync-state.json'), + ); + expect(getProjectCommitInfoPath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'project_commit_info.txt'), + ); }); - it('enables auto sync only for the explicit value 1', () => { - expect(parseAutoSyncFlag('1')).toEqual({ enabled: true }); - expect(parseAutoSyncFlag('true')).toEqual({ - enabled: false, - reason: 'invalid', - message: '[auto-sync] AUTO_UPDATE_AND_ANALYZE_FLAG must be 0 or 1; got "true". Auto sync is disabled.', - }); - }); - - it('loads sync_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { + it('loads watch_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { await fs.writeFile( - path.join(gitnexusHome, 'sync_config.yml'), + path.join(gitnexusHome, 'watch_config.yml'), [ 'sync_interval_minutes: 10', + 'max_concurrency: 3', + 'repo_git_timeout: 12s', + 'analyze_failure_threshold: 2', 'projects:', ' - local_path: /tmp/repos', - ' gitnexus_group: back_end', - ' branch: test, master, test', + ' group_name: back_end', + ' branches: [test, master, test]', ' remote_urls:', ' - git@gitee.com:qts_server/qts_account.git', ].join('\n'), @@ -73,49 +73,97 @@ describe('auto-sync', () => { expect(loaded.ok).toBe(true); if (!loaded.ok) throw new Error('expected config to load'); - expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'sync_config.yml')); + expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'watch_config.yml')); expect(loaded.config.syncIntervalMinutes).toBe(10); + expect(loaded.config.maxConcurrency).toBe(3); + expect(loaded.config.repoGitTimeoutMs).toBe(12_000); + expect(loaded.config.analyzeFailureThreshold).toBe(2); expect(loaded.config.projects[0]).toMatchObject({ localPath: '/tmp/repos', - gitnexusGroup: 'back_end', + groupName: 'back_end', branches: ['test', 'master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }); }); + it('defaults repo_git_timeout and max_concurrency and allows empty group_name', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' group_name: ""', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config'); + expect(loaded.config.repoGitTimeoutMs).toBe(10_000); + expect(loaded.config.maxConcurrency).toBe(1); + expect(loaded.config.analyzeFailureThreshold).toBe(3); + expect(loaded.config.projects[0].groupName).toBeUndefined(); + }); + + it('rejects invalid analyze_failure_threshold values', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'analyze_failure_threshold: 1', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('analyze_failure_threshold must be an integer >= 2'); + }); + it('reports missing config without throwing', async () => { const loaded = await loadAutoSyncConfig(); expect(loaded).toEqual({ ok: false, reason: 'missing', - message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'sync_config.yml')}. Auto sync is skipped.`, + message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'watch_config.yml')}. Auto sync is skipped.`, }); }); it('reports invalid config without throwing', async () => { - await fs.writeFile(path.join(gitnexusHome, 'sync_config.yml'), 'projects: []\n'); + await fs.writeFile(path.join(gitnexusHome, 'watch_config.yml'), 'projects: []\n'); const loaded = await loadAutoSyncConfig(); expect(loaded.ok).toBe(false); if (loaded.ok) throw new Error('expected invalid config'); expect(loaded.reason).toBe('invalid'); - expect(loaded.message).toContain('[auto-sync] Invalid sync_config.yml:'); + expect(loaded.message).toContain('[auto-sync] Invalid watch_config.yml:'); expect(loaded.message).toContain('sync_interval_minutes must be a positive integer'); expect(loaded.message).toContain('projects must contain at least one project'); }); it('rejects missing, relative, and traversal local_path values at config load', async () => { await fs.writeFile( - path.join(gitnexusHome, 'sync_config.yml'), + path.join(gitnexusHome, 'watch_config.yml'), [ 'sync_interval_minutes: 10', 'projects:', ' - local_path: ../repos', ' branch: master', ' remote_urls:', - ' - https://example.com/team/repo.git', + ' - git@github.com:team/repo.git', ].join('\n'), ); @@ -147,6 +195,18 @@ describe('auto-sync', () => { } }); + it('allows the default GitNexus repos directory as an auto-sync clone root', async () => { + const root = path.join(gitnexusHome, 'repos'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + }); + it('rejects symlinks in configured clone root paths', async () => { const realRoot = path.join(tempDir, 'real-root'); const linkRoot = path.join(tempDir, 'link-root'); @@ -163,33 +223,88 @@ describe('auto-sync', () => { await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( expect.objectContaining({ root, - quarantineRoot: path.join(gitnexusHome, 'quarantine'), + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), quarantineRetentionDays: 14, }), ); }); + it('creates missing configured clone roots before watch clone work', async () => { + const root = path.join(tempDir, 'missing-repos'); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + expect((await fs.stat(root)).isDirectory()).toBe(true); + }); + it('parses branch strings and arrays with trimming and de-duplication', () => { expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']); expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']); }); + it('rejects unsafe auto-sync branch names', () => { + expect(() => validateAutoSyncBranchName('feature/good-branch')).not.toThrow(); + expect(() => validateAutoSyncBranchName('-upload-pack=evil')).toThrow('must not start'); + expect(() => validateAutoSyncBranchName('feature bad')).toThrow('whitespace'); + expect(() => validateAutoSyncBranchName('feature..bad')).toThrow('must not contain ".."'); + expect(() => validateAutoSyncBranchName('bad:ref')).toThrow('not allowed'); + }); + it('extracts safe repository names from remote URLs', () => { expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( 'qts_account', ); - expect(extractRepoNameFromRemoteUrl('https://example.com/team/repo-name.git')).toBe( - 'repo-name', - ); + expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe('repo-name'); }); it('rejects unsafe repository names without sanitizing them', () => { - expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/repo$name.git')).toThrow( + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/repo$name.git')).toThrow( 'valid repository name', ); - expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/..')).toThrow( - 'valid repository name', + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/..')).toThrow('traversal'); + }); + + it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => { + expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('user@github.com:owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow('host must be'); + }); + + it('parses repo git timeout durations', () => { + expect(parseDurationMs('10s')).toBe(10_000); + expect(parseDurationMs('2m')).toBe(120_000); + expect(parseDurationMs('5000ms')).toBe(5000); + expect(parseDurationMs('10')).toBe(10_000); + expect(parseDurationMs(10)).toBe(10_000); + }); + + it('keeps branch compatibility but rejects branch and branches together', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' branches: [develop]', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('must not set both branch and branches'); }); it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => { @@ -217,6 +332,8 @@ describe('auto-sync', () => { codeCommitId: 'abc', analyzedCommitId: 'abc', lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', lastSyncTime: '2026-06-30T00:00:00.000Z', }, }, @@ -228,11 +345,13 @@ describe('auto-sync', () => { ); await expect(loadAutoSyncState(statePath)).resolves.toEqual({ '/tmp/repos/qts_account|master': { - codeCommitId: 'abc', - analyzedCommitId: 'abc', - lastAnalyzeStatus: 'success', - lastSyncTime: '2026-06-30T00:00:00.000Z', - }, + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, }); }); @@ -247,4 +366,48 @@ describe('auto-sync', () => { `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, ); }); + + it('writes project_commit_info.txt atomically', async () => { + const infoPath = path.join(tempDir, 'project_commit_info.txt'); + + await writeProjectCommitInfo( + [ + { + remoteUrl: 'git@github.com:owner/repo.git', + localPath: '/tmp/repos/repo', + branch: 'master', + codeCommitId: 'abc', + analyzedCommitId: 'abc', + status: 'success', + analyzeConsecutiveFailures: 0, + analyzeFailureThreshold: 3, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + { + remoteUrl: 'git@github.com:owner/bad.git', + localPath: '/tmp/repos/bad', + branch: 'master', + codeCommitId: 'def', + analyzedCommitId: 'abc', + status: 'threshold_skipped', + analyzeConsecutiveFailures: 3, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + ], + infoPath, + ); + + const content = await fs.readFile(infoPath, 'utf-8'); + expect(content).toContain('remote: git@github.com:owner/repo.git'); + expect(content).toContain('code_commit: abc'); + expect(content).toContain('analyze_consecutive_failures: 0'); + expect(content).toContain('analyze_failure_threshold: 3'); + expect(content).toContain('status: threshold_skipped'); + expect(content).toContain('last_analyze_error: parser crashed'); + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('project_commit_info.txt.tmp'), + ); + }); }); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 8bd6fcee9..6eb9b09ab 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -17,7 +17,11 @@ function runHelp(command: string, env: NodeJS.ProcessEnv = {}) { } function runHelpArgs(args: string[], env: NodeJS.ProcessEnv = {}) { - return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args, '--help'], { + return runCliArgs([...args, '--help'], env); +} + +function runCliArgs(args: string[], env: NodeJS.ProcessEnv = {}) { + return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args], { cwd: repoRoot, encoding: 'utf8', env: { ...process.env, ...env }, @@ -236,6 +240,42 @@ describe('CLI help surface', () => { } }); + it('watch help exposes lifecycle actions and state files', () => { + const result = runHelp('watch'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('gitnexus watch [options] [action]'); + expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch_config.yml'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/watch.pid'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/project_commit_info.txt'); + }); + + it('watch init creates the default watch_config.yml and does not overwrite it', () => { + const home = fs.mkdtempSync(path.join(repoRoot, '.tmp-test/gitnexus-watch-init-')); + try { + const first = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); + const configPath = path.join(home, 'watch_config.yml'); + + expect(first.status).toBe(0); + expect(first.stdout).toContain(`Created ${configPath}`); + const config = fs.readFileSync(configPath, 'utf8'); + expect(config).toContain('sync_interval_minutes: 10'); + expect(config).toContain('analyze_failure_threshold: 3'); + expect(config).toContain(`local_path: ${path.join(home, 'repo')}`); + expect(config).not.toContain('/abs/path/to/repos'); + expect(config).toContain('git@github.com:owner/repo.git'); + + const second = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); + + expect(second.status).toBe(1); + expect(second.stderr).toContain(`Config already exists: ${configPath}`); + expect(fs.readFileSync(configPath, 'utf8')).toBe(config); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it('wiki help shows provider, review, and verbose flags', () => { const result = runHelp('wiki'); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 9d879a22a..c16c7eb2a 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -16,20 +16,24 @@ vi.mock('../../src/core/logger.js', () => ({ import { extractRepoName, + extractWebRepoName, getCloneDir, validateGitUrl, cloneOrPull, buildCloneArgs, + buildBranchCloneArgs, buildGitEnv, normalizeGitUrlForCompare, assertRemoteMatchesRequestedUrl, isAzureDevOpsUrl, warnIfInsecureAzureConfig, + runGitForTest, } from '../../src/server/git-clone.js'; import path from 'node:path'; import os from 'node:os'; import fs from 'node:fs/promises'; import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; import { getRemoteOriginUrl } from '../../src/storage/git.js'; import { getGlobalDir } from '../../src/storage/repo-manager.js'; @@ -365,6 +369,20 @@ describe('git-clone', () => { expect(args.some((a) => a.toLowerCase().includes('authorization'))).toBe(false); expect(args.some((a) => a.includes('extraHeader'))).toBe(false); }); + + it('adds --branch before the URL separator for branch-specific clones', () => { + const args = buildBranchCloneArgs('git@github.com:owner/repo.git', '/safe/target', 'develop'); + expect(args).toEqual([ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@github.com:owner/repo.git', + '/safe/target', + ]); + }); }); describe('buildGitEnv — token injection', () => { @@ -551,6 +569,57 @@ describe('git-clone', () => { ); }); + it('keeps regular cloneOrPull restricted to http and https URLs', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('git@github.com:owner/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('Invalid URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('allows auto-sync SSH SCP clone URLs with a per-repo timeout', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const runGitForTest = vi.fn(async () => { + await fs.mkdir(target); + }); + try { + await expect( + cloneOrPull('git@gitlab.com:group/subgroup/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'develop', + runGitForTest, + }), + ).resolves.toBe(target); + + expect(runGitForTest).toHaveBeenCalledWith( + [ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@gitlab.com:group/subgroup/repo.git', + target, + ], + undefined, + { token: undefined, url: 'git@gitlab.com:group/subgroup/repo.git', timeoutMs: 10_000 }, + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('allows an explicitly controlled auto-sync clone root outside the default root', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); try { @@ -810,6 +879,30 @@ describe('git-clone', () => { }); }); + describe('extractWebRepoName — API clone compatibility', () => { + it('sanitizes repo names with spaces and unsafe directory characters at the web boundary', () => { + expect( + extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces'), + ).toBe('My_Repo_With_Spaces'); + expect(extractWebRepoName('https://example.com/team/repo$name.git')).toBe('repo_name'); + }); + + it('keeps Windows reserved names from becoming clone directories', () => { + expect(() => extractWebRepoName('https://example.com/team/CON.git')).toThrow( + 'valid repository name', + ); + expect(() => extractWebRepoName('https://example.com/team/NUL.txt')).toThrow( + 'valid repository name', + ); + }); + + it('leaves strict extractRepoName behavior unchanged for internal callers', () => { + expect(() => extractRepoName('https://example.com/team/repo$name.git')).toThrow( + 'valid repository name', + ); + }); + }); + describe('validateGitUrl — Azure DevOps URLs', () => { it('allows self-hosted Azure DevOps Server URLs', () => { expect(() => @@ -988,4 +1081,41 @@ describe('git-clone', () => { } }); }); + + describe('runGit timeout', () => { + it('waits for close and sends SIGKILL after the grace period before returning timeout', async () => { + vi.useFakeTimers(); + try { + const child = new EventEmitter() as EventEmitter & { + stderr: EventEmitter; + kill: ReturnType; + }; + child.stderr = new EventEmitter(); + child.kill = vi.fn(); + const spawnForTest = vi.fn(() => child) as unknown as typeof spawn; + + const promise = runGitForTest(['clone'], undefined, { + timeoutMs: 20, + timeoutKillGraceMs: 20, + spawnForTest, + }); + + await vi.advanceTimersByTimeAsync(25); + let settled = false; + promise.catch(() => {}).finally(() => { + settled = true; + }); + await vi.runAllTicks(); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(25); + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + child.emit('close', null); + await expect(promise).rejects.toThrow('timed out after 20ms'); + } finally { + vi.useRealTimers(); + } + }); + }); }); From 60b00c3d748c9b7be2680812614997412eaf9be4 Mon Sep 17 00:00:00 2001 From: weiyf Date: Wed, 1 Jul 2026 18:16:20 +0800 Subject: [PATCH 03/14] adds an opt-in auto sync and analysis loop for GitNexus,gitnexus watch [init|start|restart|stop|status] --- gitnexus/src/cli/watch.ts | 99 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 gitnexus/src/cli/watch.ts diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts new file mode 100644 index 000000000..ea548580e --- /dev/null +++ b/gitnexus/src/cli/watch.ts @@ -0,0 +1,99 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + getAutoSyncConfigPath, + readAutoSyncWatchStatus, + startAutoSyncWatch, + stopAutoSyncWatch, + type WatchStatusRecord, +} from '../core/auto-sync/index.js'; + +export async function watchCommand(action = 'start'): Promise { + if (action === 'init') { + await initWatchConfig(); + return; + } + if (action === 'status') { + printStatus(await readAutoSyncWatchStatus()); + return; + } + if (action === 'stop') { + await stopAutoSyncWatch(); + return; + } + if (action === 'restart') { + const stopped = await stopAutoSyncWatch(); + if (!stopped) { + process.exitCode = 1; + return; + } + await startWatchProcess(); + return; + } + if (action !== 'start') { + process.stderr.write(`[auto-sync] Unknown watch action: ${action}\n`); + process.exitCode = 1; + return; + } + await startWatchProcess(); +} + +async function startWatchProcess(): Promise { + const handle = await startAutoSyncWatch(); + if (!handle) { + process.exitCode = 1; + return; + } + + const stop = () => { + void handle.stop().finally(() => { + process.stderr.write('[auto-sync] Watch stopped.\n'); + process.exit(0); + }); + }; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); +} + +function printStatus(status: WatchStatusRecord): void { + const parts = [`state=${status.state}`]; + if (status.pid) parts.push(`pid=${status.pid}`); + if (status.configPath) parts.push(`config=${status.configPath}`); + if (status.message) parts.push(`message=${status.message}`); + parts.push(`updated_at=${status.updatedAt}`); + process.stdout.write(`${parts.join(' ')}\n`); +} + +async function initWatchConfig(): Promise { + const configPath = getAutoSyncConfigPath(); + try { + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, defaultSyncConfig(path.resolve(path.dirname(configPath), 'repo')), { + flag: 'wx', + }); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'EEXIST') { + process.stderr.write(`[auto-sync] Config already exists: ${configPath}\n`); + process.exitCode = 1; + return; + } + throw err; + } + process.stdout.write(`[auto-sync] Created ${configPath}\n`); +} + +function defaultSyncConfig(localPath: string): string { + return [ + 'sync_interval_minutes: 10', + 'max_concurrency: 1', + 'repo_git_timeout: 10s', + 'analyze_failure_threshold: 3', + 'projects:', + ` - local_path: ${localPath}`, + ' branches: [master, main]', + ' group_name: back_end', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + '', + ].join('\n'); +} From 9f49eea84ed9b4cb237b07a67dcfdb64f8190b04 Mon Sep 17 00:00:00 2001 From: weiyf Date: Mon, 6 Jul 2026 15:51:15 +0800 Subject: [PATCH 04/14] fix: address PR review cleanup --- eslint-rules/require-safe-parse.mjs | 6 +-- gitnexus/src/cli/watch.ts | 10 ++-- gitnexus/src/core/auto-sync/config.ts | 7 ++- gitnexus/src/core/auto-sync/path-security.ts | 17 ++++-- gitnexus/src/core/auto-sync/runner.ts | 22 ++++++-- gitnexus/src/core/auto-sync/starter.ts | 46 +++++++++------- gitnexus/src/core/auto-sync/state.ts | 7 ++- gitnexus/src/server/git-clone.ts | 6 +-- gitnexus/test/unit/auto-sync-runner.test.ts | 57 +++++++++++++------- gitnexus/test/unit/auto-sync.test.ts | 48 ++++++++++------- gitnexus/test/unit/git-clone.test.ts | 24 +++++---- gitnexus/test/unit/hooks.test.ts | 30 ++++++++--- 12 files changed, 184 insertions(+), 96 deletions(-) diff --git a/eslint-rules/require-safe-parse.mjs b/eslint-rules/require-safe-parse.mjs index 4ab9dbd8a..4bad4280d 100644 --- a/eslint-rules/require-safe-parse.mjs +++ b/eslint-rules/require-safe-parse.mjs @@ -19,14 +19,14 @@ * * False-positive suppression: * - Skips calls whose receiver is a known non-tree-sitter library (`JSON`, - * `URL`, `marked`, `Number`). + * `URL`, `marked`, `Number`, `path`). * - Skips calls whose first argument is a string-literal (grammar-load smoke * tests like `_testParser.parse('service X { rpc Y (R) returns (R); }')`). * - Skips test files (`.test.ts`/`.test.tsx`/`.spec.ts`). * - Skips the `safe-parse.ts` helper itself. */ -const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math']); +const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math', 'path']); export default { meta: { @@ -74,7 +74,7 @@ export default { // Receiver-text-shape skip: anything matching well-known JS APIs that // happen to have a `.parse()` shape but aren't tree-sitter. if ( - /^(JSON|URL|marked|Number|Math|Date|globalThis\.JSON)\b/.test(receiverText) || + /^(JSON|URL|marked|Number|Math|Date|path|globalThis\.JSON)\b/.test(receiverText) || /\bjson\.parse\b/i.test(receiverText) ) { return; diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts index ea548580e..c995ccf6b 100644 --- a/gitnexus/src/cli/watch.ts +++ b/gitnexus/src/cli/watch.ts @@ -68,9 +68,13 @@ async function initWatchConfig(): Promise { const configPath = getAutoSyncConfigPath(); try { await fs.mkdir(path.dirname(configPath), { recursive: true }); - await fs.writeFile(configPath, defaultSyncConfig(path.resolve(path.dirname(configPath), 'repo')), { - flag: 'wx', - }); + await fs.writeFile( + configPath, + defaultSyncConfig(path.resolve(path.dirname(configPath), 'repo')), + { + flag: 'wx', + }, + ); } catch (err: unknown) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') { process.stderr.write(`[auto-sync] Config already exists: ${configPath}\n`); diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index febf8490f..95579b91a 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -209,7 +209,9 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy export function validateAutoSyncRemoteUrl(remoteUrl: string): void { const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim()); if (!match) { - throw new Error('must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git'); + throw new Error( + 'must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git', + ); } const host = match[1].toLowerCase(); const repoPath = match[2]; @@ -223,7 +225,8 @@ export function validateAutoSyncRemoteUrl(remoteUrl: string): void { export function validateAutoSyncBranchName(branch: string): void { if (!branch.trim()) throw new Error('must not be empty'); - if (/[\s\0-\x1f\x7f]/.test(branch)) throw new Error('must not contain whitespace or control characters'); + if (/[\s\0-\x1f\x7f]/.test(branch)) + throw new Error('must not contain whitespace or control characters'); if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref'); if (branch.startsWith('-')) throw new Error('must not start with "-"'); if (branch.includes('..')) throw new Error('must not contain ".."'); diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts index e8510c248..7359c4679 100644 --- a/gitnexus/src/core/auto-sync/path-security.ts +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -65,7 +65,11 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise { +export async function quarantineAutoSyncPartial( + targetDir: string, + quarantineRoot: string, +): Promise { await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); const base = path.basename(targetDir); const stamp = new Date().toISOString().replace(/[:.]/g, '-'); @@ -119,7 +126,8 @@ function assertNotDangerousRoot(root: string): void { throw new Error(`Refusing unsafe auto-sync clone root under ${dangerousRoot}: ${root}`); } } - if (path.parse(root).root === root) throw new Error(`Refusing filesystem root as clone root: ${root}`); + if (path.parse(root).root === root) + throw new Error(`Refusing filesystem root as clone root: ${root}`); } function assertNotGitNexusInternalRoot(root: string): void { @@ -151,7 +159,8 @@ async function assertNoSymlinkPath(root: string): Promise { if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; throw err; } - if (stat.isSymbolicLink()) throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); + if (stat.isSymbolicLink()) + throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); } } diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index 55e4e75ab..558896290 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -83,7 +83,10 @@ export async function runAutoSyncOnce( const groupsToSync = new Set(); const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; const commitInfoEntries: ProjectCommitInfoEntry[] = []; - const actualConcurrency = resolveActualConcurrency(config.maxConcurrency, deps.getAvailableMemoryGB()); + const actualConcurrency = resolveActualConcurrency( + config.maxConcurrency, + deps.getAvailableMemoryGB(), + ); logger.info( `[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`, ); @@ -182,7 +185,9 @@ export async function runAutoSyncOnce( lastSyncTime, }; } catch (err: unknown) { - logger.error(`[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`); + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`, + ); return { kind: 'failed' as const, project: item.project, @@ -258,7 +263,9 @@ export async function runAutoSyncOnce( groupMembershipOk = true; } catch (err: unknown) { result.failed += 1; - logger.error(`[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`); + logger.error( + `[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`, + ); } if (groupMembershipOk && repoResult.analyzeStatus === 'success') { groupsToSync.add(repoResult.project.groupName); @@ -321,7 +328,10 @@ export function resolveActualConcurrency(configured: number, availableMemoryGB: return Math.max(1, Math.min(configured, memoryLimit)); } -async function buildWorkItems(config: AutoSyncConfig, deps: AutoSyncRunDeps): Promise { +async function buildWorkItems( + config: AutoSyncConfig, + deps: AutoSyncRunDeps, +): Promise { const items: AutoSyncWorkItem[] = []; const targetOwners = new Map(); for (const project of config.projects) { @@ -332,7 +342,9 @@ async function buildWorkItems(config: AutoSyncConfig, deps: AutoSyncRunDeps): Pr const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); const previous = targetOwners.get(targetDir); if (previous !== undefined) { - throw new Error(`Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`); + throw new Error( + `Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`, + ); } targetOwners.set(targetDir, remoteUrl); } catch (err: unknown) { diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index 8d4b00a16..15bb06295 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -48,15 +48,17 @@ export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWat }; } -export async function startAutoSyncWatch(options: { - setIntervalFn?: typeof setInterval; - clearIntervalFn?: typeof clearInterval; - runOnce?: typeof runAutoSyncOnce; - stderr?: Pick; - keepAlive?: boolean; - paths?: AutoSyncWatchPaths; - deps?: Partial; -} = {}): Promise { +export async function startAutoSyncWatch( + options: { + setIntervalFn?: typeof setInterval; + clearIntervalFn?: typeof clearInterval; + runOnce?: typeof runAutoSyncOnce; + stderr?: Pick; + keepAlive?: boolean; + paths?: AutoSyncWatchPaths; + deps?: Partial; + } = {}, +): Promise { const stderr = options.stderr ?? process.stderr; const paths = options.paths ?? getAutoSyncWatchPaths(); const deps = resolveWatchDeps(options.deps); @@ -199,13 +201,15 @@ async function acquireWatchLock( } } -export async function stopAutoSyncWatch(options: { - paths?: AutoSyncWatchPaths; - stderr?: Pick; - deps?: Partial; - timeoutMs?: number; - pollMs?: number; -} = {}): Promise { +export async function stopAutoSyncWatch( + options: { + paths?: AutoSyncWatchPaths; + stderr?: Pick; + deps?: Partial; + timeoutMs?: number; + pollMs?: number; + } = {}, +): Promise { const stderr = options.stderr ?? process.stderr; const paths = options.paths ?? getAutoSyncWatchPaths(); const deps = resolveWatchDeps(options.deps); @@ -428,14 +432,20 @@ async function readStatusFile(statusPath: string): Promise { +async function writeWatchStatus( + paths: AutoSyncWatchPaths, + record: WatchStatusRecord, +): Promise { await fs.mkdir(path.dirname(paths.statusPath), { recursive: true }); const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`; await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8'); await fs.rename(tmpPath, paths.statusPath); } -async function cleanupWatchFiles(paths: AutoSyncWatchPaths, lockHandle?: fs.FileHandle): Promise { +async function cleanupWatchFiles( + paths: AutoSyncWatchPaths, + lockHandle?: fs.FileHandle, +): Promise { await lockHandle?.close().catch(() => {}); await removeIfExists(paths.pidPath); await removeIfExists(paths.lockPath); diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts index 0de053ca9..5c4d1e39c 100644 --- a/gitnexus/src/core/auto-sync/state.ts +++ b/gitnexus/src/core/auto-sync/state.ts @@ -106,7 +106,12 @@ export interface ProjectCommitInfoEntry { branch?: string; codeCommitId?: string; analyzedCommitId?: string; - status: AutoSyncAnalyzeStatus | 'sync_failed' | 'branch_skipped' | 'branch_unavailable' | 'sync_timeout'; + status: + | AutoSyncAnalyzeStatus + | 'sync_failed' + | 'branch_skipped' + | 'branch_unavailable' + | 'sync_timeout'; analyzeConsecutiveFailures?: number; analyzeFailureThreshold?: number; lastAnalyzeError?: string; diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 60f0b44d6..5553d115b 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -790,11 +790,7 @@ export function buildGitEnv( // host-scoped Authorization header (GitHub PAT for github.com, else the // server's AZURE_DEVOPS_PAT for Azure hosts) via the GIT_CONFIG_* protocol — // never in argv. See resolveGitCredential / buildExtraHeaderKey. -function runGit( - args: string[], - cwd?: string, - options?: RunGitOptions, -): Promise { +function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { return new Promise((resolve, reject) => { const spawnGit = options?.spawnForTest ?? spawn; const proc = spawnGit('git', args, { diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 9e8c5b13e..ada907d7c 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -13,7 +13,11 @@ import { startAutoSyncWatch, stopAutoSyncWatch, } from '../../src/core/auto-sync/index.js'; -import type { AutoSyncConfig, AutoSyncRunDeps, AutoSyncWatchPaths } from '../../src/core/auto-sync/index.js'; +import type { + AutoSyncConfig, + AutoSyncRunDeps, + AutoSyncWatchPaths, +} from '../../src/core/auto-sync/index.js'; const config: AutoSyncConfig = { configPath: '/tmp/.gitnexus/watch_config.yml', @@ -44,7 +48,11 @@ function withCloneRoot(deps: Partial): Partial }; } -async function writeWatchOwner(paths: AutoSyncWatchPaths, pid: number, ownerId = `owner-${pid}`): Promise { +async function writeWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, + ownerId = `owner-${pid}`, +): Promise { await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); await fs.writeFile(paths.pidPath, `${pid}\n`); await fs.writeFile( @@ -473,7 +481,9 @@ describe('auto-sync runner', () => { }), ); expect(errorLogger).toHaveBeenCalledWith( - expect.stringContaining('Repository sync failed for git@gitee.com:qts_server/failing_sync.git'), + expect.stringContaining( + 'Repository sync failed for git@gitee.com:qts_server/failing_sync.git', + ), ); expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), @@ -540,7 +550,9 @@ describe('auto-sync runner', () => { return remoteUrl.includes('/one.git') ? '/tmp/repos/one' : '/tmp/repos/two'; }), getCurrentBranch: vi.fn(() => 'master'), - getCurrentCommit: vi.fn((repoPath) => (repoPath.endsWith('/one') ? 'one-commit' : 'two-commit')), + getCurrentCommit: vi.fn((repoPath) => + repoPath.endsWith('/one') ? 'one-commit' : 'two-commit', + ), runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'repo'), loadState: vi.fn(async () => ({})), @@ -792,17 +804,10 @@ describe('auto-sync runner', () => { await fs.mkdir(groupDir, { recursive: true }); await fs.writeFile( path.join(groupDir, 'group.yaml'), - [ - 'version: 1', - 'name: back_end', - 'repos:', - ' hr/hiring/backend: qts_account', - ].join('\n'), + ['version: 1', 'name: back_end', 'repos:', ' hr/hiring/backend: qts_account'].join('\n'), ); - await expect(addRepoToGroup({ groupName: 'back_end' }, 'qts_account')).resolves.toBe( - false, - ); + await expect(addRepoToGroup({ groupName: 'back_end' }, 'qts_account')).resolves.toBe(false); await expect(fs.readFile(path.join(groupDir, 'group.yaml'), 'utf-8')).resolves.toContain( 'hr/hiring/backend: qts_account', @@ -854,7 +859,9 @@ describe('auto-sync starter', () => { expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); expect(timer.unref).toHaveBeenCalled(); await vi.waitFor(() => { - expect(stderr.write).toHaveBeenCalledWith(expect.stringContaining('[auto-sync] Watch loop started at ')); + expect(stderr.write).toHaveBeenCalledWith( + expect.stringContaining('[auto-sync] Watch loop started at '), + ); expect(stderr.write).toHaveBeenCalledWith( '[auto-sync] Watch loop finished: synced=0 analyzed=0 skipped=0 failed=0.\n', ); @@ -935,7 +942,9 @@ describe('auto-sync starter', () => { }); expect(handle).toBeNull(); - expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch is already running with pid 12345.\n', + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -1014,7 +1023,9 @@ describe('auto-sync starter', () => { }); expect(handle).toBeNull(); - expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch is already running with pid 12345.\n', + ); expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); await expect(fs.access(paths.pidPath)).rejects.toThrow(); } finally { @@ -1073,7 +1084,11 @@ describe('auto-sync starter', () => { stopAutoSyncWatch({ paths, stderr: { write: vi.fn() }, - deps: { isProcessAlive: vi.fn(() => false), killProcess: vi.fn(), sleep: vi.fn(async () => {}) }, + deps: { + isProcessAlive: vi.fn(() => false), + killProcess: vi.fn(), + sleep: vi.fn(async () => {}), + }, }), ).resolves.toBe(false); @@ -1146,7 +1161,9 @@ describe('auto-sync starter', () => { }), ).resolves.toBe(false); - await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + await expect( + readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + ).resolves.toMatchObject({ state: 'stopping', pid: 12345, message: expect.stringContaining('did not exit'), @@ -1182,7 +1199,9 @@ describe('auto-sync starter', () => { ).resolves.toBe(false); expect(killProcess).not.toHaveBeenCalled(); - await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + await expect( + readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + ).resolves.toMatchObject({ state: 'error', pid: 12345, message: expect.stringContaining('owner'), diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index cc8be6e17..896613a90 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -176,14 +176,14 @@ describe('auto-sync', () => { it('hard-fails unsafe configured clone roots', async () => { await expect(resolveConfiguredCloneRoot('/')).rejects.toThrow('unsafe auto-sync clone root'); - await expect(resolveConfiguredCloneRoot(os.homedir())).rejects.toThrow('unsafe auto-sync clone root'); - await expect(resolveConfiguredCloneRoot(path.join(await fs.realpath(os.tmpdir()), 'repos'))).rejects.toThrow( + await expect(resolveConfiguredCloneRoot(os.homedir())).rejects.toThrow( 'unsafe auto-sync clone root', ); + await expect( + resolveConfiguredCloneRoot(path.join(await fs.realpath(os.tmpdir()), 'repos')), + ).rejects.toThrow('unsafe auto-sync clone root'); const root = path.join(tempDir, 'repos'); - await expect(resolveConfiguredCloneRoot(`${root}/../repos`)).rejects.toThrow( - 'normalized', - ); + await expect(resolveConfiguredCloneRoot(`${root}/../repos`)).rejects.toThrow('normalized'); }); it('rejects GitNexus internal directory descendants as clone roots', async () => { @@ -258,7 +258,9 @@ describe('auto-sync', () => { expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( 'qts_account', ); - expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe('repo-name'); + expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe( + 'repo-name', + ); }); it('rejects unsafe repository names without sanitizing them', () => { @@ -271,11 +273,19 @@ describe('auto-sync', () => { it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => { expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow(); expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow(); - expect(() => validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git')).not.toThrow(); - expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow('must use'); - expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow('must use'); + expect(() => + validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git'), + ).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow( + 'must use', + ); + expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow( + 'must use', + ); expect(() => validateAutoSyncRemoteUrl('user@github.com:owner/repo.git')).toThrow('must use'); - expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow('host must be'); + expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow( + 'host must be', + ); }); it('parses repo git timeout durations', () => { @@ -318,9 +328,7 @@ describe('auto-sync', () => { previousStatus: 'failed', }), ).toBe(true); - expect(shouldAnalyzeCommit({ currentCommit: 'def', previousAnalyzedCommit: 'abc' })).toBe( - true, - ); + expect(shouldAnalyzeCommit({ currentCommit: 'def', previousAnalyzedCommit: 'abc' })).toBe(true); }); it('saves state atomically and reloads it', async () => { @@ -345,13 +353,13 @@ describe('auto-sync', () => { ); await expect(loadAutoSyncState(statePath)).resolves.toEqual({ '/tmp/repos/qts_account|master': { - codeCommitId: 'abc', - analyzedCommitId: 'abc', - lastAnalyzeStatus: 'success', - analyzeConsecutiveFailures: 2, - lastAnalyzeError: 'old error', - lastSyncTime: '2026-06-30T00:00:00.000Z', - }, + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, }); }); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index c16c7eb2a..11ec037e6 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -672,7 +672,9 @@ describe('git-clone', () => { try { await new Promise((resolve, reject) => { const proc = spawn('git', ['init'], { cwd: root, stdio: 'ignore' }); - proc.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`git init ${code}`)))); + proc.on('close', (code) => + code === 0 ? resolve() : reject(new Error(`git init ${code}`)), + ); proc.on('error', reject); }); await fs.rename(path.join(root, '.git'), path.join(target, '.git')).catch(async () => { @@ -719,9 +721,9 @@ describe('git-clone', () => { ).rejects.toThrow('git clone failed'); const entries = await fs.readdir(quarantineRoot); - expect(entries.some((entry) => entry.startsWith('auto-sync-') && entry.endsWith('-repo'))).toBe( - true, - ); + expect( + entries.some((entry) => entry.startsWith('auto-sync-') && entry.endsWith('-repo')), + ).toBe(true); } finally { await fs.rm(root, { recursive: true, force: true }); } @@ -881,9 +883,9 @@ describe('git-clone', () => { describe('extractWebRepoName — API clone compatibility', () => { it('sanitizes repo names with spaces and unsafe directory characters at the web boundary', () => { - expect( - extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces'), - ).toBe('My_Repo_With_Spaces'); + expect(extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces')).toBe( + 'My_Repo_With_Spaces', + ); expect(extractWebRepoName('https://example.com/team/repo$name.git')).toBe('repo_name'); }); @@ -1102,9 +1104,11 @@ describe('git-clone', () => { await vi.advanceTimersByTimeAsync(25); let settled = false; - promise.catch(() => {}).finally(() => { - settled = true; - }); + promise + .catch(() => {}) + .finally(() => { + settled = true; + }); await vi.runAllTicks(); expect(child.kill).toHaveBeenCalledWith('SIGTERM'); expect(settled).toBe(false); diff --git a/gitnexus/test/unit/hooks.test.ts b/gitnexus/test/unit/hooks.test.ts index 91f18232c..3c2c9d250 100644 --- a/gitnexus/test/unit/hooks.test.ts +++ b/gitnexus/test/unit/hooks.test.ts @@ -411,14 +411,32 @@ describe('windowsHide regression', () => { /** * Count spawn-family invocations. The regex matches ``spawn(``, * ``spawnSync(``, ``execFile(``, ``execFileSync(``, - * ``execFileAsync(``, ``execSync(`` as function calls — not - * destructures (``const { spawn } = ...``), not method calls - * (``.exec(``), not bare ``exec()`` (which collides with regex - * ``.exec()``; we explicitly drop it). + * ``execFileAsync(``, ``execSync(`` and simple local aliases that + * point at one of those functions as function calls — not destructures + * (``const { spawn } = ...``), not method calls (``.exec(``), not bare + * ``exec()`` (which collides with regex ``.exec()``; we explicitly + * drop it). */ function countSpawnCalls(codeSource: string): number { - const re = - /(^|[^a-zA-Z0-9_$.])(spawn|spawnSync|execFile|execFileSync|execFileAsync|execSync)\s*\(/gm; + const spawnFunctions = [ + 'spawn', + 'spawnSync', + 'execFile', + 'execFileSync', + 'execFileAsync', + 'execSync', + ]; + const spawnNames = new Set(spawnFunctions); + const aliasRe = new RegExp( + `\\bconst\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*[^;\\n]*\\b(?:${spawnFunctions.join('|')})\\b`, + 'g', + ); + let aliasMatch: RegExpExecArray | null; + while ((aliasMatch = aliasRe.exec(codeSource)) !== null) { + spawnNames.add(aliasMatch[1]); + } + + const re = new RegExp(`(^|[^a-zA-Z0-9_$.])(${[...spawnNames].join('|')})\\s*\\(`, 'gm'); let count = 0; while (re.exec(codeSource) !== null) { count++; From d68999bf578addf5578c8ecfe5a501195b95b848 Mon Sep 17 00:00:00 2001 From: weiyf Date: Fri, 17 Jul 2026 11:47:24 +0800 Subject: [PATCH 05/14] Prettier code style --- README.md | 73 +++++++++++++++++++++++----------------------- gitnexus/README.md | 8 ++--- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index dbb203b64..63618d01c 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,15 @@ https://github.com/user-attachments/assets/172685ba-8e54-4ea7-9ad1-e31a3398da72 ## Two Ways to Use GitNexus -| | **CLI + MCP** | **Web UI** | -| ----------- | --------------------------------------------------------------------- | -------------------------------------------------------------------- | -| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | +| | **CLI + MCP** | **Web UI** | +| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | | **For** | Daily development with Cursor, Claude Code, Antigravity, Codex, Windsurf, OpenCode | Quick exploration, demos, one-off analysis | -| **Scale** | Full repos, any size | Limited by browser memory (~5k files), or unlimited via backend mode | -| **Install** | `npm install -g gitnexus` | No install — [gitnexus.vercel.app](https://gitnexus.vercel.app) | -| **Storage** | LadybugDB native (fast, persistent) | LadybugDB WASM (in-memory, per session) | -| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Privacy** | Everything local, no network | Everything in-browser, no server | +| **Scale** | Full repos, any size | Limited by browser memory (~5k files), or unlimited via backend mode | +| **Install** | `npm install -g gitnexus` | No install — [gitnexus.vercel.app](https://gitnexus.vercel.app) | +| **Storage** | LadybugDB native (fast, persistent) | LadybugDB WASM (in-memory, per session) | +| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | +| **Privacy** | Everything local, no network | Everything in-browser, no server | > **Bridge mode:** `gitnexus serve` connects the two — the web UI auto-detects the local server and can browse all your CLI-indexed repos without re-uploading or re-indexing. @@ -127,18 +127,19 @@ To configure MCP for your editor, run `npx gitnexus setup` once — or set it up ### Editor Support -| Editor | MCP | Skills | Hooks (auto-augment) | Support | -| -------------------- | --- | ------ | --------------------------------------------------------------------------------------- | ------------ | -| **Claude Code** | Yes | Yes | Yes (PreToolUse + PostToolUse) | **Full** | -| **Cursor** | Yes | Yes | Yes (postToolUse, [manual install](gitnexus-cursor-integration/README.md#hook-install)) | **Full** | -| **Antigravity** (Google) | Yes | Yes | Yes (AfterTool, [Gemini CLI hooks schema](https://geminicli.com/docs/hooks/reference/))[¹](#fn-antigravity-hooks) | **Full** | -| **Codex** | Yes | Yes | — | MCP + Skills | -| **Windsurf** | Yes | — | — | MCP | -| **OpenCode** | Yes | Yes | — | MCP + Skills | +| Editor | MCP | Skills | Hooks (auto-augment) | Support | +| ------------------------ | --- | ------ | ----------------------------------------------------------------------------------------------------------------- | ------------ | +| **Claude Code** | Yes | Yes | Yes (PreToolUse + PostToolUse) | **Full** | +| **Cursor** | Yes | Yes | Yes (postToolUse, [manual install](gitnexus-cursor-integration/README.md#hook-install)) | **Full** | +| **Antigravity** (Google) | Yes | Yes | Yes (AfterTool, [Gemini CLI hooks schema](https://geminicli.com/docs/hooks/reference/))[¹](#fn-antigravity-hooks) | **Full** | +| **Codex** | Yes | Yes | — | MCP + Skills | +| **Windsurf** | Yes | — | — | MCP | +| **OpenCode** | Yes | Yes | — | MCP + Skills | > **Claude Code** gets the deepest integration: MCP tools + agent skills + PreToolUse hooks that enrich searches with graph context + PostToolUse hooks that detect a stale index after commits and prompt the agent to reindex. + > ¹ **Antigravity hooks** follow the [Gemini CLI hooks reference](https://geminicli.com/docs/hooks/reference/) (Antigravity 2.0 is the documented successor to Gemini CLI). Augmentation runs in `AfterTool` because `BeforeTool` has no context-injection channel in the Gemini contract — the agent sees graph context appended to the tool result via `hookSpecificOutput.additionalContext`. Stale-index hints land in the same channel after a successful `git commit/merge/rebase/cherry-pick/pull`. The schema may evolve if Antigravity-specific hook docs diverge from Gemini CLI's; the implementation will track those changes. ## Community Integrations @@ -318,7 +319,7 @@ Commit a `.gitnexusrc` JSON file at the repo root to preconfigure recurring `ana "skipContextFiles": true, // alias of skipAgentsMd: keep your own AGENTS.md/CLAUDE.md "skipSkills": true, // don't install .claude/skills/gitnexus/ "embeddings": true, // generate embeddings by default - "workerTimeout": 60 + "workerTimeout": 60, } ``` @@ -339,25 +340,25 @@ Notes: Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max-file-size`, `--verbose`). Use the env-var form when you'd otherwise repeat the same flag every run, or when invoking GitNexus from a long-running host (MCP server, eval-server, CI shell) that already manages its own environment. CLI flags take precedence over env vars; env vars take precedence over built-in defaults. -| Variable | Default | Effect | Tune when… | -| -------------------------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | -| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | -| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | -| `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | -| `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | -| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | -| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | -| `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | -| `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | -| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | -| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | -| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | -| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Combined with `timeoutBackoffFactor`, prevents exponentially-growing retries from stalling for hours. | Slow files that legitimately need long total retry windows; lower to fail-fast on stalls. | -| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD`| `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | -| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | -| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | -| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | +| Variable | Default | Effect | Tune when… | +| ----------------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | +| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | +| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | +| `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | +| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | +| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | +| `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | +| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | +| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Combined with `timeoutBackoffFactor`, prevents exponentially-growing retries from stalling for hours. | Slow files that legitimately need long total retry windows; lower to fail-fast on stalls. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | +| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | +| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | #### Publishing to understand-quickly (opt-in) diff --git a/gitnexus/README.md b/gitnexus/README.md index 2f30a5e84..447889f7b 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -385,7 +385,7 @@ Configure the behavior with two environment variables: | -------------------------------------------- | ---------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | | `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process `INSTALL` child before it is killed. | -| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | +| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | ```bash @@ -456,9 +456,9 @@ Three env vars expose the pool's resilience layers (respawn budget, cumulative-t After scope resolution, analyze prunes inert block-local value symbols (a function-local `const`/`let`/`var` that ends up with only its structural `File→DEFINES` edge) to keep the graph focused on cross-symbol relationships. Module/file-scope symbols, class members, and any local with a real edge are always kept. -| Variable | Default | Effect | -| ------------------------------------ | ------- | ------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` | unset | Set to `1`/`true` to keep inert block-local value symbols instead of pruning them. | +| Variable | Default | Effect | +| ----------------------------------- | ------- | ---------------------------------------------------------------------------------- | +| `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` | unset | Set to `1`/`true` to keep inert block-local value symbols instead of pruning them. | Programmatic callers can pass `keepLocalValueSymbols: true` in `PipelineOptions` instead of setting the env var. From ac5923716ef1a94f0cffcb086dd05e46bb9b689b Mon Sep 17 00:00:00 2001 From: weiyf Date: Fri, 17 Jul 2026 16:06:42 +0800 Subject: [PATCH 06/14] merge main --- README.md | 37 ++++++++++++++++++++++- gitnexus/README.md | 4 +-- gitnexus/test/unit/cli-index-help.test.ts | 7 +---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c56e64256..bc931afbe 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,6 @@ gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1 # auto-sized to the repo). 0 is rejected — there is no sequential mode. gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) -gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml ``` If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `--worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. @@ -415,6 +414,42 @@ If embeddings are skipped on a large repository, the indexed graph likely exceed +
+Keep remote repositories indexed with gitnexus watch + +`gitnexus watch` clones or pulls configured repositories, analyzes new commits, and optionally syncs their group. It runs once immediately, then repeats on the configured interval. It runs in the foreground; use your process manager if it must survive a shell session. + +```bash +# 1. Create the config once. It never overwrites an existing file. +gitnexus watch init + +# 2. Edit $GITNEXUS_HOME/watch_config.yml, then start it. +gitnexus watch start # `gitnexus watch` is equivalent +gitnexus watch status +gitnexus watch restart # Required after config changes +gitnexus watch stop +``` + +`GITNEXUS_HOME` defaults to `~/.gitnexus`. A minimal configuration: + +```yaml +sync_interval_minutes: 10 +projects: + - local_path: /absolute/path/to/clones + branches: [main, master] + remote_urls: + - git@github.com:owner/repo.git +``` + +- `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. +- Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. +- `branches` are tried in order. The legacy `branch` field is supported, but do not set both. +- Add `group_name` only after creating that group with `gitnexus group create `. + +See the [full watch configuration and runtime reference](gitnexus/README.md#gitnexus-watch) for concurrency, timeouts, failure thresholds, and runtime files. + +
+
Repository groups (multi-repo / monorepo service tracking) diff --git a/gitnexus/README.md b/gitnexus/README.md index 7695b176e..1510a2475 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -277,7 +277,7 @@ gitnexus group impact --target --repo # Cross-repo ### `gitnexus watch` -`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output. +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus watch init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.lock` prevents multiple watch processes for one home, `watch.pid` and `watch.status.json` expose process state, and `quarantine/` stores partial clone output. ```yaml sync_interval_minutes: 10 @@ -294,7 +294,7 @@ projects: - git@gitee.com:owner/repo.git ``` -`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project. `GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `GITNEXUS_HOME/watch/auto-sync-state.json`. +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index a6897b953..d6737950f 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -20,16 +20,11 @@ function runHelpArgs(args: string[], env: NodeJS.ProcessEnv = {}) { } function runCliArgs(args: string[], env: NodeJS.ProcessEnv = {}) { - return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args], { + return spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, ...args], { cwd: repoRoot, encoding: 'utf8', env: { ...process.env, ...env }, }); - // return spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, ...args, '--help'], { - // cwd: repoRoot, - // encoding: 'utf8', - // env: { ...process.env, ...env }, - // }); } function runRootHelp(env: NodeJS.ProcessEnv = {}) { From ac2acab2d06669b0238f4cf058ff25279e7a3155 Mon Sep 17 00:00:00 2001 From: weiyf Date: Mon, 20 Jul 2026 13:17:22 +0800 Subject: [PATCH 07/14] fix(watch): protect local repos and cancel active analysis --- README.md | 3 + gitnexus/README.md | 24 +- gitnexus/src/cli/watch.ts | 2 + .../core/auto-sync/analysis-worker-launch.ts | 120 ++++++++ gitnexus/src/core/auto-sync/config.ts | 39 ++- gitnexus/src/core/auto-sync/index.ts | 1 + gitnexus/src/core/auto-sync/runner.ts | 265 ++++++++++------- gitnexus/src/core/auto-sync/starter.ts | 81 +++++- gitnexus/src/core/auto-sync/state.ts | 29 +- gitnexus/src/server/git-clone.ts | 64 ++++- .../unit/auto-sync-analysis-worker.test.ts | 54 ++++ gitnexus/test/unit/auto-sync-runner.test.ts | 270 ++++++++++++++++-- gitnexus/test/unit/auto-sync.test.ts | 64 ++++- gitnexus/test/unit/cli-index-help.test.ts | 2 + gitnexus/test/unit/git-clone.test.ts | 94 ++++++ 15 files changed, 940 insertions(+), 172 deletions(-) create mode 100644 gitnexus/src/core/auto-sync/analysis-worker-launch.ts create mode 100644 gitnexus/test/unit/auto-sync-analysis-worker.test.ts diff --git a/README.md b/README.md index 54d8e1ff1..4a1ecd3de 100644 --- a/README.md +++ b/README.md @@ -440,9 +440,11 @@ gitnexus watch stop ```yaml sync_interval_minutes: 10 +analyze_timeout: 5m projects: - local_path: /absolute/path/to/clones branches: [main, master] + overwrite_local_changes: false remote_urls: - git@github.com:owner/repo.git ``` @@ -450,6 +452,7 @@ projects: - `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. - Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. - `branches` are tried in order. The legacy `branch` field is supported, but do not set both. +- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten. Stopping watch cancels an active analysis immediately. - Add `group_name` only after creating that group with `gitnexus group create `. See the [full watch configuration and runtime reference](gitnexus/README.md#gitnexus-watch) for concurrency, timeouts, failure thresholds, and runtime files. diff --git a/gitnexus/README.md b/gitnexus/README.md index 275e0bc41..5c3f15cf4 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -283,18 +283,20 @@ gitnexus group impact --target --repo # Cross-repo sync_interval_minutes: 10 max_concurrency: 1 repo_git_timeout: 10s +analyze_timeout: 5m analyze_failure_threshold: 3 projects: - local_path: /abs/path/to/repos branches: [master, main] group_name: back_end + overwrite_local_changes: false remote_urls: - git@github.com:owner/repo.git - git@gitlab.com:group/repo.git - git@gitee.com:owner/repo.git ``` -`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. On timeout watch terminates that worker, records the failed attempt, and resumes scheduling only after the worker exits. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes. Stopping watch cancels an active analysis worker immediately. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit resets the failure count and is analyzed again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. @@ -488,16 +490,16 @@ GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnex Configure the behavior with these environment variables: -| Variable | Values | Default | Effect | -| -------------------------------------------- | ------------------------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | -| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | -| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | -| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | -| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` uses the bundled default path. `icebug` and `auto` currently behave identically: both try the experimental Icebug CSR path and fall back to Graphology if the optional native module is unavailable or incompatible. | -| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. | -| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | +| Variable | Values | Default | Effect | +| -------------------------------------------- | ------------------------------ | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | +| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | +| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | +| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | +| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` uses the bundled default path. `icebug` and `auto` currently behave identically: both try the experimental Icebug CSR path and fall back to Graphology if the optional native module is unavailable or incompatible. | +| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. | +| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | ```bash # Offline/airgapped: never reach the network for extensions diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts index c995ccf6b..41dc2ba59 100644 --- a/gitnexus/src/cli/watch.ts +++ b/gitnexus/src/cli/watch.ts @@ -91,11 +91,13 @@ function defaultSyncConfig(localPath: string): string { 'sync_interval_minutes: 10', 'max_concurrency: 1', 'repo_git_timeout: 10s', + 'analyze_timeout: 5m', 'analyze_failure_threshold: 3', 'projects:', ` - local_path: ${localPath}`, ' branches: [master, main]', ' group_name: back_end', + ' overwrite_local_changes: false', ' remote_urls:', ' - git@github.com:owner/repo.git', '', diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts new file mode 100644 index 000000000..8b8d492b2 --- /dev/null +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -0,0 +1,120 @@ +import { fork, type ChildProcess } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; +import type { WorkerMessage } from '../../server/analyze-worker.js'; + +const _require = createRequire(import.meta.url); +const TERMINATION_GRACE_MS = 10_000; + +export type AutoSyncAnalysisRunner = ( + repoPath: string, + options: AnalyzeOptions, + timeoutMs: number, + signal?: AbortSignal, +) => Promise>; + +interface AnalysisWorker extends Pick {} + +export interface AutoSyncAnalysisLaunchDeps { + forkWorker: (workerPath: string, execArgv: string[]) => AnalysisWorker; + setTimeoutFn: typeof setTimeout; + clearTimeoutFn: typeof clearTimeout; +} + +const DEFAULT_DEPS: AutoSyncAnalysisLaunchDeps = { + forkWorker: (workerPath, execArgv) => + fork(workerPath, [], { + execArgv, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }), + setTimeoutFn: setTimeout, + clearTimeoutFn: clearTimeout, +}; + +export function createAutoSyncAnalysisRunner( + overrides: Partial = {}, +): AutoSyncAnalysisRunner { + const deps = { ...DEFAULT_DEPS, ...overrides }; + return (repoPath, options, timeoutMs, signal) => + new Promise>((resolve, reject) => { + if (signal?.aborted) { + reject(new Error('Analysis cancelled.')); + return; + } + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const workerPath = path.join( + path.dirname(callerPath), + '../../server', + isDev ? 'analyze-worker.ts' : 'analyze-worker.js', + ); + if (!existsSync(workerPath)) { + reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); + return; + } + const execArgv = isDev + ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href, '--max-old-space-size=8192'] + : ['--max-old-space-size=8192']; + const child = deps.forkWorker(workerPath, execArgv); + let outcome: WorkerMessage | undefined; + let timedOut = false; + let cancelled = false; + let terminationGrace: ReturnType | undefined; + const timeout = deps.setTimeoutFn(() => { + timedOut = true; + child.kill('SIGTERM'); + terminationGrace = deps.setTimeoutFn(() => child.kill('SIGKILL'), TERMINATION_GRACE_MS); + }, timeoutMs); + const onAbort = () => { + cancelled = true; + deps.clearTimeoutFn(timeout); + if (terminationGrace) deps.clearTimeoutFn(terminationGrace); + child.kill('SIGKILL'); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + + child.on('message', (message: WorkerMessage) => { + if (message.type !== 'progress') outcome ??= message; + else outcome = message; + }); + child.on('error', (error) => { + outcome = { type: 'error', message: `Auto-sync analyze worker error: ${error.message}` }; + }); + child.on('exit', (code, childSignal) => { + deps.clearTimeoutFn(timeout); + if (terminationGrace) deps.clearTimeoutFn(terminationGrace); + signal?.removeEventListener('abort', onAbort); + if (cancelled) { + reject(new Error('Analysis cancelled.')); + return; + } + if (timedOut) { + reject( + new Error( + `Analysis timed out after ${timeoutMs}ms and worker exited (${childSignal ?? code ?? 'unknown'}).`, + ), + ); + return; + } + if (outcome?.type === 'complete') { + resolve({ stats: outcome.result.stats }); + return; + } + if (outcome?.type === 'error') { + reject(new Error(outcome.message)); + return; + } + reject( + new Error( + `Auto-sync analyze worker exited before completion (${signal ?? code ?? 'unknown'}).`, + ), + ); + }); + child.send({ type: 'start', repoPath, options }); + }); +} + +export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner(); diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 95579b91a..7096b1cbc 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -10,6 +10,8 @@ const yaml = _require('js-yaml') as typeof import('js-yaml'); export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml'; const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; const MIN_SYNC_INTERVAL_MINUTES = 5; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_SYNC_INTERVAL_MINUTES = Math.floor(MAX_TIMER_DELAY_MS / 60_000); const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000; const DEFAULT_MAX_CONCURRENCY = 1; export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3; @@ -19,6 +21,7 @@ const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']); export interface AutoSyncProjectConfig { localPath: string; groupName?: string; + overwriteLocalChanges: boolean; branches: string[]; remoteUrls: string[]; } @@ -27,6 +30,7 @@ export interface AutoSyncConfig { configPath: string; syncIntervalMinutes: number; repoGitTimeoutMs: number; + analyzeTimeoutMs: number; maxConcurrency: number; analyzeFailureThreshold: number; projects: AutoSyncProjectConfig[]; @@ -100,6 +104,8 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy errors.push('sync_interval_minutes must be a positive integer'); } else if (interval < MIN_SYNC_INTERVAL_MINUTES) { errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); + } else if (interval > MAX_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must not exceed ${MAX_SYNC_INTERVAL_MINUTES}`); } const maxConcurrency = @@ -116,6 +122,24 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy errors.push('repo_git_timeout must be a positive duration such as 10s'); } + const maxAnalyzeTimeoutMs = + Number.isInteger(interval) && + interval >= MIN_SYNC_INTERVAL_MINUTES && + interval <= MAX_SYNC_INTERVAL_MINUTES + ? interval * 30_000 + : undefined; + const analyzeTimeoutMs = + raw.analyze_timeout === undefined + ? (maxAnalyzeTimeoutMs ?? 0) + : parseDurationMs(raw.analyze_timeout); + if (!Number.isInteger(analyzeTimeoutMs) || analyzeTimeoutMs <= 0) { + errors.push('analyze_timeout must be a positive duration such as 30m'); + } else if (maxAnalyzeTimeoutMs !== undefined && analyzeTimeoutMs > maxAnalyzeTimeoutMs) { + errors.push( + `analyze_timeout must not exceed half of sync_interval_minutes (${maxAnalyzeTimeoutMs / 60_000}m)`, + ); + } + const analyzeFailureThreshold = raw.analyze_failure_threshold === undefined ? DEFAULT_ANALYZE_FAILURE_THRESHOLD @@ -189,8 +213,20 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy errors.push(`projects[${index}].group_name is invalid`); } + const overwriteLocalChanges = + project.overwrite_local_changes === undefined ? false : project.overwrite_local_changes; + if (typeof overwriteLocalChanges !== 'boolean') { + errors.push(`projects[${index}].overwrite_local_changes must be a boolean`); + } + if (localPath && remoteUrls.length > 0 && branches.length > 0) { - projects.push({ localPath, groupName, branches, remoteUrls }); + projects.push({ + localPath, + groupName, + overwriteLocalChanges: overwriteLocalChanges === true, + branches, + remoteUrls, + }); } }); } @@ -200,6 +236,7 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy configPath, syncIntervalMinutes: interval, repoGitTimeoutMs, + analyzeTimeoutMs, maxConcurrency, analyzeFailureThreshold, projects, diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts index 6dc23a177..4182843bf 100644 --- a/gitnexus/src/core/auto-sync/index.ts +++ b/gitnexus/src/core/auto-sync/index.ts @@ -34,6 +34,7 @@ export { } from './path-security.js'; export { addRepoToGroup, + getAutoSyncRepoIdentity, getConfiguredRepoPath, resolveActualConcurrency, runAutoSyncOnce, diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index 558896290..9f18d39a3 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -22,6 +22,7 @@ import { } from './state.js'; import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js'; import { validateAutoSyncRemoteUrl } from './config.js'; +import { runAutoSyncAnalysis, type AutoSyncAnalysisRunner } from './analysis-worker-launch.js'; export interface AutoSyncLogger { info(message: string): void; @@ -33,7 +34,8 @@ export interface AutoSyncRunDeps { cloneOrPull: typeof cloneOrPull; getCurrentBranch: typeof getCurrentBranch; getCurrentCommit: typeof getCurrentCommit; - runFullAnalysis: typeof runFullAnalysis; + runFullAnalysis?: typeof runFullAnalysis; + runAnalysis: AutoSyncAnalysisRunner; registerRepo: typeof registerRepo; loadState: typeof loadAutoSyncState; saveState: typeof saveAutoSyncState; @@ -61,7 +63,7 @@ const DEFAULT_DEPS: AutoSyncRunDeps = { cloneOrPull, getCurrentBranch, getCurrentCommit, - runFullAnalysis, + runAnalysis: runAutoSyncAnalysis, registerRepo, loadState: loadAutoSyncState, saveState: saveAutoSyncState, @@ -74,12 +76,19 @@ const DEFAULT_DEPS: AutoSyncRunDeps = { export async function runAutoSyncOnce( config: AutoSyncConfig, - options: { deps?: Partial; logger?: AutoSyncLogger; now?: () => Date } = {}, + options: { + deps?: Partial; + logger?: AutoSyncLogger; + now?: () => Date; + signal?: AbortSignal; + } = {}, ): Promise { const deps = { ...DEFAULT_DEPS, ...options.deps }; const logger = options.logger ?? DEFAULT_LOGGER; const now = options.now ?? (() => new Date()); + throwIfAborted(options.signal); const state = await deps.loadState(); + throwIfAborted(options.signal); const groupsToSync = new Set(); const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; const commitInfoEntries: ProjectCommitInfoEntry[] = []; @@ -92,112 +101,135 @@ export async function runAutoSyncOnce( ); const workItems = await buildWorkItems(config, deps); - const repoResults = await mapWithConcurrency(workItems, actualConcurrency, async (item) => { - const lastSyncTime = now().toISOString(); - try { - validateAutoSyncRemoteUrl(item.remoteUrl); - const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl); - const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName); - const syncResult = await syncFirstAvailableBranch({ - item, - repoName, - targetDir, - timeoutMs: config.repoGitTimeoutMs, - deps, - logger, - }); - if (syncResult.ok === false) { + const repoResults = await mapWithConcurrency( + workItems, + actualConcurrency, + options.signal, + async (item) => { + const lastSyncTime = now().toISOString(); + try { + throwIfAborted(options.signal); + validateAutoSyncRemoteUrl(item.remoteUrl); + const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName); + const syncResult = await syncFirstAvailableBranch({ + item, + repoName, + targetDir, + timeoutMs: config.repoGitTimeoutMs, + deps, + logger, + }); + throwIfAborted(options.signal); + if (syncResult.ok === false) { + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir, + branch: item.project.branches[0], + status: syncResult.status, + analyzeConsecutiveFailures: 0, + lastSyncTime, + }; + } + + const currentBranch = syncResult.branch; + + const currentCommit = deps.getCurrentCommit(targetDir); + const stateKey = buildStateKey(targetDir, currentBranch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; + let lastAnalyzeError = previous?.lastAnalyzeError; + let stats: RepoMeta['stats'] | undefined; + + if (previous && previous.codeCommitId !== currentCommit) { + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } + + if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { + analyzeStatus = 'threshold_skipped'; + logger.error( + `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, + ); + } else if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = deps.runFullAnalysis + ? await deps.runFullAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ) + : await deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + ); + throwIfAborted(options.signal); + stats = analysis.stats; + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; + analyzeStatus = 'failed'; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = shortErrorMessage(err); + logger.error( + `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, + ); + } + } else { + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + throwIfAborted(options.signal); + + return { + kind: 'synced' as const, + project: item.project, + repoName, + remoteUrl: item.remoteUrl, + targetDir, + branch: currentBranch, + currentCommit, + analyzedCommitId, + analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + stats, + stateKey, + lastSyncTime, + }; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; logger.error( - `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + `[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`, ); return { kind: 'failed' as const, project: item.project, remoteUrl: item.remoteUrl, - targetDir, - branch: item.project.branches[0], - status: syncResult.status, - analyzeConsecutiveFailures: 0, + targetDir: '', + status: 'sync_failed' as const, lastSyncTime, }; } - - const currentBranch = syncResult.branch; - - const currentCommit = deps.getCurrentCommit(targetDir); - const stateKey = buildStateKey(targetDir, currentBranch); - const previous = state[stateKey]; - let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; - let analyzedCommitId = previous?.analyzedCommitId; - let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; - let lastAnalyzeError = previous?.lastAnalyzeError; - let stats: RepoMeta['stats'] | undefined; - - if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { - analyzeStatus = 'threshold_skipped'; - logger.error( - `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, - ); - } else if ( - shouldAnalyzeCommit({ - currentCommit, - previousAnalyzedCommit: previous?.analyzedCommitId, - previousStatus: previous?.lastAnalyzeStatus, - }) - ) { - try { - const analysis = await deps.runFullAnalysis( - targetDir, - { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, - { onProgress: () => {} }, - ); - stats = analysis.stats; - analyzeStatus = 'success'; - analyzedCommitId = currentCommit; - analyzeConsecutiveFailures = 0; - lastAnalyzeError = undefined; - } catch (err: unknown) { - analyzeStatus = 'failed'; - analyzeConsecutiveFailures += 1; - lastAnalyzeError = shortErrorMessage(err); - logger.error( - `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, - ); - } - } else { - logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); - } - - return { - kind: 'synced' as const, - project: item.project, - repoName, - remoteUrl: item.remoteUrl, - targetDir, - branch: currentBranch, - currentCommit, - analyzedCommitId, - analyzeStatus, - analyzeConsecutiveFailures, - lastAnalyzeError, - stats, - stateKey, - lastSyncTime, - }; - } catch (err: unknown) { - logger.error( - `[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`, - ); - return { - kind: 'failed' as const, - project: item.project, - remoteUrl: item.remoteUrl, - targetDir: '', - status: 'sync_failed' as const, - lastSyncTime, - }; - } - }); + }, + ); for (const repoResult of repoResults) { if (repoResult.kind === 'failed') { @@ -231,8 +263,7 @@ export async function runAutoSyncOnce( branch: repoResult.branch, }; await deps.registerRepo(repoResult.targetDir, meta, { - name: repoResult.repoName, - allowDuplicateName: true, + name: getAutoSyncRepoIdentity(repoResult.remoteUrl), }); result.analyzed += 1; } else if (repoResult.analyzeStatus === 'failed') { @@ -259,7 +290,11 @@ export async function runAutoSyncOnce( if (repoResult.project.groupName) { let groupMembershipOk = false; try { - await deps.addRepoToGroup(repoResult.project, repoResult.repoName); + await deps.addRepoToGroup( + repoResult.project, + getAutoSyncRepoIdentity(repoResult.remoteUrl), + getAutoSyncRepoIdentity(repoResult.remoteUrl), + ); groupMembershipOk = true; } catch (err: unknown) { result.failed += 1; @@ -300,17 +335,27 @@ export function getConfiguredRepoPath( export async function addRepoToGroup( project: Pick, - repoName: string, + groupPath: string, + registryName = groupPath, ): Promise { if (!project.groupName) return false; const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName); const config = await loadGroupConfig(groupDir); - if (Object.values(config.repos).includes(repoName)) return false; - config.repos[repoName] = repoName; + if (config.repos[groupPath] === registryName) return false; + if (config.repos[groupPath] !== undefined) { + throw new Error(`group path ${groupPath} is already mapped to ${config.repos[groupPath]}`); + } + config.repos[groupPath] = registryName; await writeGroupConfigAtomic(path.join(groupDir, 'group.yaml'), config); return true; } +export function getAutoSyncRepoIdentity(remoteUrl: string): string { + validateAutoSyncRemoteUrl(remoteUrl); + const [, host, remotePath] = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim())!; + return `${host.toLowerCase()}/${remotePath.replace(/\.git$/, '')}`; +} + export async function syncGroupByName(groupName: string): Promise { const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName); const config = await loadGroupConfig(groupDir); @@ -359,21 +404,28 @@ async function buildWorkItems( async function mapWithConcurrency( items: T[], concurrency: number, + signal: AbortSignal | undefined, worker: (item: T) => Promise, ): Promise { const results: R[] = new Array(items.length); let nextIndex = 0; const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { while (nextIndex < items.length) { + throwIfAborted(signal); const currentIndex = nextIndex; nextIndex += 1; results[currentIndex] = await worker(items[currentIndex]); + throwIfAborted(signal); } }); await Promise.all(runners); return results; } +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new Error('Auto-sync run cancelled.'); +} + interface AutoSyncWorkItem { project: AutoSyncProjectConfig; remoteUrl: string; @@ -402,6 +454,7 @@ async function syncFirstAvailableBranch(input: { allowAutoSyncSsh: true, timeoutMs: input.timeoutMs, branch, + overwriteLocalChanges: input.item.project.overwriteLocalChanges, }); const currentBranch = input.deps.getCurrentBranch(input.targetDir); if (currentBranch === branch) return { ok: true, branch }; diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index 15bb06295..bad0c901b 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises'; import crypto from 'node:crypto'; import path from 'node:path'; +import { execFileSync } from 'node:child_process'; import { getGlobalDir } from '../../storage/repo-manager.js'; import { loadAutoSyncConfig } from './config.js'; import { runAutoSyncOnce } from './runner.js'; @@ -35,6 +36,7 @@ export interface AutoSyncWatchPaths { export interface AutoSyncWatchControlDeps { isProcessAlive(pid: number): boolean; + readProcessCommand(pid: number): string | undefined; killProcess(pid: number, signal?: NodeJS.Signals): void; sleep(ms: number): Promise; } @@ -94,16 +96,17 @@ export async function startAutoSyncWatch( }); const runOnce = options.runOnce ?? runAutoSyncOnce; - let running = false; + let activeRun: Promise | undefined; + let activeAbortController: AbortController | undefined; const runSafely = () => { - if (running) { + if (activeRun) { stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); return; } - running = true; const startedAt = new Date(); stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); - void runOnce(loaded.config) + const abortController = new AbortController(); + const run = runOnce(loaded.config, { signal: abortController.signal }) .then((result) => { stderr.write( `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, @@ -112,10 +115,15 @@ export async function startAutoSyncWatch( .catch((err: unknown) => { stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); stderr.write('[auto-sync] Watch loop finished: failed.\n'); - }) - .finally(() => { - running = false; }); + activeRun = run; + activeAbortController = abortController; + void run.finally(() => { + if (activeRun === run) { + activeRun = undefined; + activeAbortController = undefined; + } + }); }; runSafely(); @@ -127,6 +135,15 @@ export async function startAutoSyncWatch( return { stop: async () => { clearIntervalFn(timer); + activeAbortController?.abort(); + await writeWatchStatus(paths, { + state: 'stopping', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + await activeRun?.catch(() => {}); await writeWatchStatus(paths, { state: 'stopped', pid: process.pid, @@ -162,6 +179,18 @@ async function acquireWatchLock( } if (deps.isProcessAlive(lock.pid)) { + const reason = getWatchProcessIdentityError(lock.pid, deps); + if (reason) { + stderr.write(`[auto-sync] Refusing to trust existing watch pid ${lock.pid}; ${reason}.\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: lock.pid, + ownerId: lock.ownerId, + message: reason, + updatedAt: new Date().toISOString(), + }); + return null; + } stderr.write(`[auto-sync] Watch is already running with pid ${lock.pid}.\n`); await writeWatchStatus(paths, { state: 'running', @@ -272,7 +301,7 @@ export async function stopAutoSyncWatch( }); return false; } - const owner = await readVerifiedWatchOwner(paths, pid); + const owner = await readVerifiedWatchOwner(paths, pid, deps); if (owner.ok === false) { const message = `refusing to stop pid ${pid}; ${owner.reason}`; stderr.write(`[auto-sync] ${message}.\n`); @@ -334,7 +363,10 @@ export async function readAutoSyncWatchStatus( } if (pid) { const stored = await readStatusFile(paths.statusPath); - const owner = await readVerifiedWatchOwner(paths, pid); + if (stored?.state === 'error') { + return { ...stored, pid, updatedAt: new Date().toISOString() }; + } + const owner = await readVerifiedWatchOwner(paths, pid, resolvedDeps); if (owner.ok === false) { return { ...stored, @@ -380,6 +412,7 @@ async function readLockFile(lockPath: string): Promise { const [status, lock] = await Promise.all([ readStatusFile(paths.statusPath), @@ -392,9 +425,26 @@ async function readVerifiedWatchOwner( if (!status.ownerId || status.ownerId !== lock.ownerId) { return { ok: false, reason: 'watch status owner does not match lock owner' }; } + const identityError = getWatchProcessIdentityError(pid, deps); + if (identityError) return { ok: false, reason: identityError }; return { ok: true, owner: lock }; } +function getWatchProcessIdentityError( + pid: number, + deps: AutoSyncWatchControlDeps, +): string | undefined { + const command = deps.readProcessCommand(pid); + if (!command) return 'unable to verify process command'; + if ( + !/(?:^|\s)watch(?:\s|$)/.test(command) || + !/(?:gitnexus|[\\/]cli[\\/]index\.(?:ts|[cm]?js))/.test(command) + ) { + return 'pid command is not a GitNexus watch process'; + } + return undefined; +} + async function waitForProcessExit( pid: number, options: { deps: AutoSyncWatchControlDeps; timeoutMs: number; pollMs: number }, @@ -474,6 +524,19 @@ function resolveWatchDeps(deps: Partial = {}): AutoSyn return false; } }), + readProcessCommand: + deps.readProcessCommand ?? + ((pid) => { + try { + const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return command || undefined; + } catch { + return undefined; + } + }), killProcess: deps.killProcess ?? ((pid, signal = 'SIGTERM') => { diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts index 5c4d1e39c..f38fce3f9 100644 --- a/gitnexus/src/core/auto-sync/state.ts +++ b/gitnexus/src/core/auto-sync/state.ts @@ -47,9 +47,12 @@ export async function loadAutoSyncState( try { const raw = await fs.readFile(statePath, 'utf-8'); const parsed = JSON.parse(raw); - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as AutoSyncCommitState) - : {}; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}; + return Object.fromEntries( + Object.entries(parsed).filter((entry): entry is [string, AutoSyncCommitStateEntry] => + isAutoSyncCommitStateEntry(entry[1]), + ), + ); } catch (err: unknown) { if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { process.stderr.write( @@ -60,6 +63,26 @@ export async function loadAutoSyncState( } } +function isAutoSyncCommitStateEntry(value: unknown): value is AutoSyncCommitStateEntry { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const entry = value as Record; + return ( + typeof entry.codeCommitId === 'string' && + typeof entry.lastSyncTime === 'string' && + (entry.analyzedCommitId === undefined || typeof entry.analyzedCommitId === 'string') && + (entry.lastAnalyzeStatus === undefined || + entry.lastAnalyzeStatus === 'success' || + entry.lastAnalyzeStatus === 'failed' || + entry.lastAnalyzeStatus === 'skipped' || + entry.lastAnalyzeStatus === 'threshold_skipped') && + (entry.analyzeConsecutiveFailures === undefined || + (typeof entry.analyzeConsecutiveFailures === 'number' && + Number.isInteger(entry.analyzeConsecutiveFailures) && + entry.analyzeConsecutiveFailures >= 0)) && + (entry.lastAnalyzeError === undefined || typeof entry.lastAnalyzeError === 'string') + ); +} + export async function saveAutoSyncState( state: AutoSyncCommitState, statePath = getAutoSyncStatePath(), diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 5553d115b..91df78f3c 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -272,6 +272,7 @@ export interface CloneOrPullOptions { allowAutoSyncSsh?: boolean; timeoutMs?: number; branch?: string; + overwriteLocalChanges?: boolean; runGitForTest?: typeof runGit; } @@ -551,22 +552,55 @@ export async function cloneOrPull( onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); const runGitImpl = options?.runGitForTest ?? runGit; if (options?.branch) { - await runGitImpl(['fetch', '--depth', '1', 'origin', options.branch], safeTarget, { - token: options?.token, - url, - timeoutMs: options?.timeoutMs, - }); - await runGitImpl(['checkout', options.branch], safeTarget, { + if (!options.overwriteLocalChanges) { + const status = await runGitImpl(['status', '--porcelain'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + if (status.trim()) { + throw new Error( + `Refusing to update ${safeTarget}: local changes detected. Set overwrite_local_changes: true to overwrite them.`, + ); + } + } + await runGitImpl( + [ + 'fetch', + '--depth', + '1', + 'origin', + `refs/heads/${options.branch}:refs/remotes/origin/${options.branch}`, + ], + safeTarget, + { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }, + ); + await runGitImpl( + [ + 'checkout', + ...(options.overwriteLocalChanges ? ['--force'] : []), + '-B', + options.branch, + `origin/${options.branch}`, + ], + safeTarget, + { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }, + ); + } else { + await runGitImpl(['pull', '--ff-only'], safeTarget, { token: options?.token, url, timeoutMs: options?.timeoutMs, }); } - await runGitImpl(['pull', '--ff-only'], safeTarget, { - token: options?.token, - url, - timeoutMs: options?.timeoutMs, - }); } else { if (targetExists) { throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); @@ -790,7 +824,7 @@ export function buildGitEnv( // host-scoped Authorization header (GitHub PAT for github.com, else the // server's AZURE_DEVOPS_PAT for Azure hosts) via the GIT_CONFIG_* protocol — // never in argv. See resolveGitCredential / buildExtraHeaderKey. -function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { +function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { return new Promise((resolve, reject) => { const spawnGit = options?.spawnForTest ?? spawn; const proc = spawnGit('git', args, { @@ -800,6 +834,7 @@ function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise< env: buildGitEnv(process.env, options), }); + let stdout = ''; let stderr = ''; let settled = false; let timedOut = false; @@ -821,6 +856,9 @@ function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise< }, options.timeoutKillGraceMs ?? 1_000); }, options.timeoutMs) : undefined; + proc.stdout?.on('data', (chunk: Buffer) => { + stdout += chunk; + }); proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk; }); @@ -830,7 +868,7 @@ function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise< finish(() => reject(new Error(`git ${args[0]} timed out after ${options?.timeoutMs}ms`))); return; } - if (code === 0) finish(resolve); + if (code === 0) finish(() => resolve(stdout)); else { // Log full stderr internally but don't expose it to API callers (SSRF mitigation) if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`); diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts new file mode 100644 index 000000000..f6e06f869 --- /dev/null +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -0,0 +1,54 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, vi } from 'vitest'; +import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; + +describe('auto-sync analysis worker', () => { + it('waits for timed-out worker exit before releasing the scheduled run', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + expect(child.send).toHaveBeenCalledWith({ + type: 'start', + repoPath: '/tmp/repo', + options: { branch: 'main' }, + }); + + timers[0](); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + timers[1](); + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + + child.emit('exit', null, 'SIGKILL'); + await expect(result).rejects.toThrow('Analysis timed out after 50ms'); + }); + + it('kills an active worker immediately when watch is stopped', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + }); + const controller = new AbortController(); + + const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); + controller.abort(); + + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + child.emit('exit', null, 'SIGKILL'); + await expect(result).rejects.toThrow('Analysis cancelled'); + }); +}); diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index ada907d7c..79501c3ed 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -23,12 +23,14 @@ const config: AutoSyncConfig = { configPath: '/tmp/.gitnexus/watch_config.yml', syncIntervalMinutes: 10, repoGitTimeoutMs: 10_000, + analyzeTimeoutMs: 1_800_000, maxConcurrency: 1, analyzeFailureThreshold: 3, projects: [ { localPath: '/tmp/repos', groupName: 'back_end', + overwriteLocalChanges: false, branches: ['master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }, @@ -40,6 +42,7 @@ const cloneRoot = { quarantineRoot: '/tmp/.gitnexus/watch/quarantine', quarantineRetentionDays: 14, }; +const verifiedWatchCommand = 'node /gitnexus/dist/cli/index.js watch'; function withCloneRoot(deps: Partial): Partial { return { @@ -112,6 +115,7 @@ describe('auto-sync runner', () => { allowAutoSyncSsh: true, timeoutMs: 10_000, branch: 'master', + overwriteLocalChanges: false, }, ); expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); @@ -123,7 +127,7 @@ describe('auto-sync runner', () => { expect(deps.registerRepo).toHaveBeenCalledWith( '/tmp/repos/qts_account', expect.objectContaining({ lastCommit: 'commit-2', branch: 'master' }), - { name: 'qts_account', allowDuplicateName: true }, + { name: 'gitee.com/qts_server/qts_account' }, ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); expect(deps.writeCommitInfo).toHaveBeenCalledWith([ @@ -156,7 +160,11 @@ describe('auto-sync runner', () => { logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, }); - expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); }); @@ -188,10 +196,79 @@ describe('auto-sync runner', () => { }); expect(result.analyzed).toBe(1); - expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); }); + it('uses distinct registry and group identities for repositories with the same basename', async () => { + const duplicateConfig: AutoSyncConfig = { + ...config, + projects: [ + { + localPath: '/tmp/repos-a', + groupName: 'back_end', + branches: ['main'], + remoteUrls: ['git@github.com:team-a/service.git'], + }, + { + localPath: '/tmp/repos-b', + groupName: 'back_end', + branches: ['main'], + remoteUrls: ['git@gitlab.com:team-b/service.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + resolveCloneRoot: vi.fn(async (localPath: string) => ({ + ...cloneRoot, + root: localPath, + })), + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'main'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'service'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(duplicateConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.registerRepo).toHaveBeenNthCalledWith( + 1, + '/tmp/repos-a/service', + expect.anything(), + { name: 'github.com/team-a/service' }, + ); + expect(deps.registerRepo).toHaveBeenNthCalledWith( + 2, + '/tmp/repos-b/service', + expect.anything(), + { name: 'gitlab.com/team-b/service' }, + ); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + duplicateConfig.projects[0], + 'github.com/team-a/service', + 'github.com/team-a/service', + ); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + duplicateConfig.projects[1], + 'gitlab.com/team-b/service', + 'gitlab.com/team-b/service', + ); + }); + it('skips analysis when commit id has not changed', async () => { const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), @@ -258,10 +335,42 @@ describe('auto-sync runner', () => { allowAutoSyncSsh: true, timeoutMs: 10_000, branch: 'master', + overwriteLocalChanges: false, }, ); }); + it('passes the watch stop signal to the isolated analysis runner', async () => { + const controller = new AbortController(); + const runAnalysis = vi.fn(async () => ({ stats: { files: 1 } }) as any); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis, + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + signal: controller.signal, + }); + + expect(runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + { branch: 'master', skipAgentsMd: true, skipSkills: true }, + 1_800_000, + controller.signal, + ); + }); + it('falls back through configured branches and analyzes the first pullable branch', async () => { const warnLogger = vi.fn(); const errorLogger = vi.fn(); @@ -470,7 +579,11 @@ describe('auto-sync runner', () => { expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 2 }); expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); expect(deps.registerRepo).not.toHaveBeenCalled(); - expect(deps.addRepoToGroup).toHaveBeenCalledWith(failingConfig.projects[0], 'qts_account'); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + failingConfig.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); expect(deps.syncGroupByName).not.toHaveBeenCalled(); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ @@ -514,7 +627,11 @@ describe('auto-sync runner', () => { }); expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); - expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Group sync failed for back_end'), @@ -673,7 +790,7 @@ describe('auto-sync runner', () => { expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ '/tmp/repos/qts_account|master': expect.objectContaining({ - analyzeConsecutiveFailures: 2, + analyzeConsecutiveFailures: 1, lastAnalyzeError: 'parser crashed with stack', lastAnalyzeStatus: 'failed', }), @@ -682,24 +799,24 @@ describe('auto-sync runner', () => { expect(deps.writeCommitInfo).toHaveBeenCalledWith([ expect.objectContaining({ status: 'failed', - analyzeConsecutiveFailures: 2, + analyzeConsecutiveFailures: 1, analyzeFailureThreshold: 3, lastAnalyzeError: 'parser crashed with stack', }), ]); expect(errorLogger).toHaveBeenCalledWith( - '[auto-sync] Analysis failed for /tmp/repos/qts_account; consecutive failures 2/3: parser crashed with stack', + '[auto-sync] Analysis failed for /tmp/repos/qts_account; consecutive failures 1/3: parser crashed with stack', ); }); - it('skips analyze when consecutive failures have reached the threshold', async () => { + it('retries analysis on a new commit after consecutive failures reached the threshold', async () => { const errorLogger = vi.fn(); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(), - registerRepo: vi.fn(), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({ '/tmp/repos/qts_account|master': { codeCommitId: 'commit-1', @@ -723,28 +840,26 @@ describe('auto-sync runner', () => { now: () => new Date('2026-06-30T00:00:00.000Z'), }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); - expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.runFullAnalysis).toHaveBeenCalledTimes(1); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ '/tmp/repos/qts_account|master': expect.objectContaining({ - analyzeConsecutiveFailures: 3, - lastAnalyzeError: 'parser crashed', - lastAnalyzeStatus: 'threshold_skipped', + analyzeConsecutiveFailures: 0, + lastAnalyzeError: undefined, + lastAnalyzeStatus: 'success', }), }), ); expect(deps.writeCommitInfo).toHaveBeenCalledWith([ expect.objectContaining({ - status: 'threshold_skipped', - analyzeConsecutiveFailures: 3, + status: 'success', + analyzeConsecutiveFailures: 0, analyzeFailureThreshold: 3, - lastAnalyzeError: 'parser crashed', + lastAnalyzeError: undefined, }), ]); - expect(errorLogger).toHaveBeenCalledWith( - '[auto-sync] Skip analysis for /tmp/repos/qts_account; analyze consecutive failures 3/3 reached threshold. Fix the repository or clear auto-sync state before retrying.', - ); + expect(errorLogger).not.toHaveBeenCalled(); }); it('clears prior analyze failure count after a successful analyze', async () => { @@ -807,7 +922,9 @@ describe('auto-sync runner', () => { ['version: 1', 'name: back_end', 'repos:', ' hr/hiring/backend: qts_account'].join('\n'), ); - await expect(addRepoToGroup({ groupName: 'back_end' }, 'qts_account')).resolves.toBe(false); + await expect( + addRepoToGroup({ groupName: 'back_end' }, 'hr/hiring/backend', 'qts_account'), + ).resolves.toBe(false); await expect(fs.readFile(path.join(groupDir, 'group.yaml'), 'utf-8')).resolves.toContain( 'hr/hiring/backend: qts_account', @@ -929,6 +1046,54 @@ describe('auto-sync starter', () => { } }); + it('cancels the active run before removing watch ownership files', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const cancelled = vi.fn(); + const runOnce = vi.fn( + (_config, options) => + new Promise((resolve) => { + options?.signal?.addEventListener( + 'abort', + () => { + cancelled(); + resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }, + { once: true }, + ); + }), + ); + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + const handle = await startAutoSyncWatch({ + runOnce, + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + const paths = getAutoSyncWatchPaths(tempDir); + await handle!.stop(); + + expect(cancelled).toHaveBeenCalledTimes(1); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it('refuses a second running watch for the same GITNEXUS_HOME', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); @@ -938,7 +1103,10 @@ describe('auto-sync starter', () => { const handle = await startAutoSyncWatch({ paths, stderr, - deps: { isProcessAlive: vi.fn(() => true) }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + }, }); expect(handle).toBeNull(); @@ -1019,7 +1187,10 @@ describe('auto-sync starter', () => { paths, stderr, runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), - deps: { isProcessAlive: vi.fn(() => true) }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + }, }); expect(handle).toBeNull(); @@ -1114,7 +1285,10 @@ describe('auto-sync starter', () => { await writeWatchOwner(paths, 12345); await expect( - readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + }), ).resolves.toMatchObject({ state: 'running', pid: 12345 }); await expect( stopAutoSyncWatch({ @@ -1123,6 +1297,7 @@ describe('auto-sync starter', () => { pollMs: 1, deps: { isProcessAlive: vi.fn(() => alive), + readProcessCommand: vi.fn(() => verifiedWatchCommand), killProcess: vi.fn((pid, signal) => { killProcess(pid, signal); alive = false; @@ -1155,6 +1330,7 @@ describe('auto-sync starter', () => { pollMs: 1, deps: { isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), killProcess: vi.fn(), sleep: vi.fn(async () => {}), }, @@ -1162,7 +1338,10 @@ describe('auto-sync starter', () => { ).resolves.toBe(false); await expect( - readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + }), ).resolves.toMatchObject({ state: 'stopping', pid: 12345, @@ -1211,6 +1390,42 @@ describe('auto-sync starter', () => { } }); + it('refuses to signal a reused pid whose command is not GitNexus watch', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + killProcess, + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(false); + + expect(killProcess).not.toHaveBeenCalled(); + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + }), + ).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('not a GitNexus watch process'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it('restart can start only after stop confirms pid and lock cleanup', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); @@ -1242,6 +1457,7 @@ describe('auto-sync starter', () => { stderr: { write: vi.fn() }, deps: { isProcessAlive: vi.fn(() => alive), + readProcessCommand: vi.fn(() => verifiedWatchCommand), killProcess: vi.fn(() => { alive = false; }), diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 896613a90..cca2365e7 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -56,13 +56,15 @@ describe('auto-sync', () => { await fs.writeFile( path.join(gitnexusHome, 'watch_config.yml'), [ - 'sync_interval_minutes: 10', + 'sync_interval_minutes: 120', 'max_concurrency: 3', 'repo_git_timeout: 12s', + 'analyze_timeout: 45m', 'analyze_failure_threshold: 2', 'projects:', ' - local_path: /tmp/repos', ' group_name: back_end', + ' overwrite_local_changes: true', ' branches: [test, master, test]', ' remote_urls:', ' - git@gitee.com:qts_server/qts_account.git', @@ -74,13 +76,15 @@ describe('auto-sync', () => { expect(loaded.ok).toBe(true); if (!loaded.ok) throw new Error('expected config to load'); expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'watch_config.yml')); - expect(loaded.config.syncIntervalMinutes).toBe(10); + expect(loaded.config.syncIntervalMinutes).toBe(120); expect(loaded.config.maxConcurrency).toBe(3); expect(loaded.config.repoGitTimeoutMs).toBe(12_000); + expect(loaded.config.analyzeTimeoutMs).toBe(2_700_000); expect(loaded.config.analyzeFailureThreshold).toBe(2); expect(loaded.config.projects[0]).toMatchObject({ localPath: '/tmp/repos', groupName: 'back_end', + overwriteLocalChanges: true, branches: ['test', 'master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }); @@ -105,9 +109,34 @@ describe('auto-sync', () => { expect(loaded.ok).toBe(true); if (!loaded.ok) throw new Error('expected config'); expect(loaded.config.repoGitTimeoutMs).toBe(10_000); + expect(loaded.config.analyzeTimeoutMs).toBe(300_000); expect(loaded.config.maxConcurrency).toBe(1); expect(loaded.config.analyzeFailureThreshold).toBe(3); expect(loaded.config.projects[0].groupName).toBeUndefined(); + expect(loaded.config.projects[0].overwriteLocalChanges).toBe(false); + }); + + it('rejects analyze_timeout values above half the sync interval', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'analyze_timeout: 6m', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain( + 'analyze_timeout must not exceed half of sync_interval_minutes (5m)', + ); }); it('rejects invalid analyze_failure_threshold values', async () => { @@ -375,6 +404,37 @@ describe('auto-sync', () => { ); }); + it('drops malformed state entries while preserving valid entries', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + await fs.writeFile( + statePath, + JSON.stringify({ + '/tmp/repos/valid|main': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 0, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + '/tmp/repos/invalid|main': { + codeCommitId: 123, + analyzeConsecutiveFailures: -1, + lastSyncTime: null, + }, + }), + ); + + await expect(loadAutoSyncState(statePath)).resolves.toEqual({ + '/tmp/repos/valid|main': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 0, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }); + }); + it('writes project_commit_info.txt atomically', async () => { const infoPath = path.join(tempDir, 'project_commit_info.txt'); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index d6737950f..97ef63567 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -268,6 +268,8 @@ describe('CLI help surface', () => { const config = fs.readFileSync(configPath, 'utf8'); expect(config).toContain('sync_interval_minutes: 10'); expect(config).toContain('analyze_failure_threshold: 3'); + expect(config).toContain('analyze_timeout: 5m'); + expect(config).toContain('overwrite_local_changes: false'); expect(config).toContain(`local_path: ${path.join(home, 'repo')}`); expect(config).not.toContain('/abs/path/to/repos'); expect(config).toContain('git@github.com:owner/repo.git'); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 11ec037e6..1b57b8347 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -52,6 +52,25 @@ async function mkControlledRoot(prefix: string): Promise { return fs.realpath(await fs.mkdtemp(path.join(base, prefix))); } +function runGit(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const proc = spawn('git', args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk; + }); + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk; + }); + proc.on('close', (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(`git ${args.join(' ')} failed (${code}): ${stderr}`)); + }); + proc.on('error', reject); + }); +} + describe('git-clone', () => { describe('extractRepoName', () => { it('extracts name from HTTPS URL', () => { @@ -588,6 +607,7 @@ describe('git-clone', () => { const target = path.join(root, 'repo'); const runGitForTest = vi.fn(async () => { await fs.mkdir(target); + return ''; }); try { await expect( @@ -702,6 +722,80 @@ describe('git-clone', () => { } }); + it('switches a shallow single-branch clone to a fallback branch', async () => { + const root = await mkControlledRoot('gitnexus-shallow-fallback-'); + const source = path.join(root, 'source'); + const remote = path.join(root, 'remote.git'); + const target = path.join(root, 'repo'); + const remoteUrl = 'git@github.com:team/repo.git'; + const gitConfig = path.join(root, 'gitconfig'); + const previousGlobalConfig = process.env.GIT_CONFIG_GLOBAL; + const previousNoSystemConfig = process.env.GIT_CONFIG_NOSYSTEM; + + try { + await runGit(['init', '--bare', remote], root); + await runGit(['init', '--initial-branch=master', source], root); + await runGit(['config', 'user.email', 'test@example.com'], source); + await runGit(['config', 'user.name', 'GitNexus Test'], source); + await fs.writeFile(path.join(source, 'branch.txt'), 'master\n'); + await runGit(['add', 'branch.txt'], source); + await runGit(['commit', '-m', 'master'], source); + await runGit(['checkout', '-b', 'main'], source); + await fs.writeFile(path.join(source, 'branch.txt'), 'main\n'); + await runGit(['commit', '-am', 'main'], source); + await runGit(['remote', 'add', 'origin', `file://${remote}`], source); + await runGit(['push', 'origin', 'master', 'main'], source); + + await fs.writeFile( + gitConfig, + `[protocol "file"]\n\tallow = always\n[url "file://${remote}"]\n\tinsteadOf = ${remoteUrl}\n`, + ); + process.env.GIT_CONFIG_GLOBAL = gitConfig; + process.env.GIT_CONFIG_NOSYSTEM = '1'; + + await runGit(['clone', '--depth', '1', '--branch', 'master', remoteUrl, target], root); + await expect( + runGit(['show-ref', '--verify', '--quiet', 'refs/remotes/origin/main'], target), + ).rejects.toThrow(); + await expect(runGit(['rev-parse', '--is-shallow-repository'], target)).resolves.toBe( + 'true\n', + ); + + await fs.writeFile(path.join(target, 'branch.txt'), 'local changes\n'); + await expect( + cloneOrPull(remoteUrl, target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + branch: 'main', + }), + ).rejects.toThrow(); + await expect(fs.readFile(path.join(target, 'branch.txt'), 'utf8')).resolves.toBe( + 'local changes\n', + ); + + await cloneOrPull(remoteUrl, target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + branch: 'main', + overwriteLocalChanges: true, + }); + + await expect(runGit(['branch', '--show-current'], target)).resolves.toBe('main\n'); + await expect(fs.readFile(path.join(target, 'branch.txt'), 'utf8')).resolves.toBe('main\n'); + await expect(runGit(['rev-parse', 'main'], target)).resolves.toBe( + await runGit(['rev-parse', 'origin/main'], target), + ); + } finally { + if (previousGlobalConfig === undefined) delete process.env.GIT_CONFIG_GLOBAL; + else process.env.GIT_CONFIG_GLOBAL = previousGlobalConfig; + if (previousNoSystemConfig === undefined) delete process.env.GIT_CONFIG_NOSYSTEM; + else process.env.GIT_CONFIG_NOSYSTEM = previousNoSystemConfig; + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('quarantines partial auto-sync clone output on clone failure', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); const quarantineRoot = path.join(root, 'quarantine'); From e92d8457961daf3d069330236588eec3b8eda291 Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 4 Aug 2026 17:34:18 +0800 Subject: [PATCH 08/14] fix(watch): harden auto-sync lifecycle and locking - validate watch process identity before lifecycle operations\n- serialize registry, analysis, and LadybugDB access with recoverable locks\n- harden clone paths, symlinks, hooks, quarantine, and worker timeouts\n- install procps in the CLI image for reliable Docker watch control\n- add focused regression coverage for lifecycle, locks, clone, and registry behavior --- .gitignore | 1 - Dockerfile.cli | 5 +- README.md | 7 +- gitnexus/README.md | 7 +- gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 2 + gitnexus/src/cli/index.ts | 13 +- gitnexus/src/cli/watch.ts | 37 +- .../core/auto-sync/analysis-worker-launch.ts | 75 ++- gitnexus/src/core/auto-sync/config.ts | 2 + gitnexus/src/core/auto-sync/index.ts | 3 + gitnexus/src/core/auto-sync/path-security.ts | 59 +- gitnexus/src/core/auto-sync/runner.ts | 138 +++-- gitnexus/src/core/auto-sync/starter.ts | 490 ++++++++-------- gitnexus/src/core/auto-sync/state.ts | 25 + gitnexus/src/core/lbug/lbug-adapter.ts | 15 +- gitnexus/src/server/analyze-worker-core.ts | 58 +- gitnexus/src/server/analyze-worker.ts | 19 +- gitnexus/src/server/git-clone.ts | 102 ++-- gitnexus/src/storage/file-lock.ts | 156 +++++ gitnexus/src/storage/repo-manager.ts | 129 +++-- gitnexus/src/utils/process-identity.ts | 34 ++ .../test/unit/analyze-worker-core.test.ts | 31 + .../unit/auto-sync-analysis-worker.test.ts | 46 ++ gitnexus/test/unit/auto-sync-runner.test.ts | 537 +++++++++++++----- gitnexus/test/unit/auto-sync.test.ts | 111 ++++ gitnexus/test/unit/cli-index-help.test.ts | 54 +- gitnexus/test/unit/file-lock.test.ts | 154 +++++ gitnexus/test/unit/git-clone.test.ts | 168 +++++- gitnexus/test/unit/process-identity.test.ts | 22 + gitnexus/test/unit/repo-manager.test.ts | 19 + gitnexus/test/unit/watch-command.test.ts | 43 ++ 32 files changed, 1883 insertions(+), 681 deletions(-) create mode 100644 gitnexus/src/storage/file-lock.ts create mode 100644 gitnexus/src/utils/process-identity.ts create mode 100644 gitnexus/test/unit/file-lock.test.ts create mode 100644 gitnexus/test/unit/process-identity.test.ts create mode 100644 gitnexus/test/unit/watch-command.test.ts diff --git a/.gitignore b/.gitignore index 794ed8145..ffde3bb06 100644 --- a/.gitignore +++ b/.gitignore @@ -128,7 +128,6 @@ local_docs/ !.agents/plugins/marketplace.json .context/ gitnexus/web/ -/log/ # Machine-local skill-evolution evidence (consumed by eval/workflow_bench/evolve.py) eval/workflow_bench/learnings.jsonl diff --git a/Dockerfile.cli b/Dockerfile.cli index 633d23f5d..365d0e539 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -51,8 +51,9 @@ RUN npm run postinstall --prefix gitnexus # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -# curl for the healthcheck; git for cloning; ca-certificates for TLS verification. -RUN apt-get update && apt-get install -y --no-install-recommends curl git ca-certificates && rm -rf /var/lib/apt/lists/* \ +# curl for the healthcheck; git for cloning; procps for watch process identity; +# ca-certificates for TLS verification. +RUN apt-get update && apt-get install -y --no-install-recommends curl git procps ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/local/lib/node_modules/npm \ && rm -rf /usr/local/lib/node_modules/corepack \ && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack diff --git a/README.md b/README.md index 4a1ecd3de..a89c4d74b 100644 --- a/README.md +++ b/README.md @@ -434,6 +434,7 @@ gitnexus watch start # `gitnexus watch` is equivalent gitnexus watch status gitnexus watch restart # Required after config changes gitnexus watch stop +gitnexus watch reset # Clear failure state; leaves clones and indexes intact ``` `GITNEXUS_HOME` defaults to `~/.gitnexus`. A minimal configuration: @@ -449,11 +450,11 @@ projects: - git@github.com:owner/repo.git ``` -- `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. +- `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. Clones are stored below it as `host/namespace/repo`. - Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. - `branches` are tried in order. The legacy `branch` field is supported, but do not set both. -- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten. Stopping watch cancels an active analysis immediately. -- Add `group_name` only after creating that group with `gitnexus group create `. +- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. Timed-out workers are terminated before scheduling resumes. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten. Stopping watch cancels an active analysis immediately. +- Add `group_name` only after creating that group with `gitnexus group create `. Partial clone output is isolated and removed after 14 days. See the [full watch configuration and runtime reference](gitnexus/README.md#gitnexus-watch) for concurrency, timeouts, failure thresholds, and runtime files. diff --git a/gitnexus/README.md b/gitnexus/README.md index a9bf37417..15153cf1f 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -240,7 +240,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) -gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml +gitnexus watch [init|start|restart|stop|status|reset] # Control auto-sync from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus index # Register an existing .gitnexus/ folder into the global registry @@ -277,7 +277,7 @@ gitnexus group impact --target --repo # Cross-repo ### `gitnexus watch` -`gitnexus watch` is the explicit long-running auto-sync entrypoint. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus watch init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.lock` prevents multiple watch processes for one home, `watch.pid` and `watch.status.json` expose process state, and `quarantine/` stores partial clone output. +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus watch init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, `status`, and `reset` manage the same `GITNEXUS_HOME` instance. `reset` removes only the derived analysis state and commit snapshot; clones, indexes, and registry entries are untouched. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.mutex` prevents multiple watch processes for one home, `watch.owner.json` records ownership metadata, `watch.pid` plus `watch.status.json` expose process state, and `quarantine/` stores partial clone output before entries are removed after 14 days. Mutexes with verified dead owners are reclaimed automatically after an abnormal exit. Invalid or legacy mutexes fail closed; confirm no watch process is running before manually removing `watch.mutex` and stale `watch.pid` / `watch.owner.json`. ```yaml sync_interval_minutes: 10 @@ -288,7 +288,6 @@ analyze_failure_threshold: 3 projects: - local_path: /abs/path/to/repos branches: [master, main] - group_name: back_end overwrite_local_changes: false remote_urls: - git@github.com:owner/repo.git @@ -296,7 +295,7 @@ projects: - git@gitee.com:owner/repo.git ``` -`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. On timeout watch terminates that worker, records the failed attempt, and resumes scheduling only after the worker exits. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes. Stopping watch cancels an active analysis worker immediately. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit resets the failure count and is analyzed again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal; each remote is cloned below it as `host/namespace/repo`, preventing same-basename repositories from colliding. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. On timeout watch terminates that worker, records the failed attempt, and resumes scheduling only after the worker exits. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes. Stopping watch cancels an active analysis worker immediately. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit or `gitnexus watch reset` clears the block and allows analysis again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 94fa3c8be..8d8a05097 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -126,6 +126,8 @@ export const en = { 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', 'help.command.watch.description': 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + 'help.watch.details': + '\nActions: init, start (default), restart, stop, status, reset\nConfiguration: GITNEXUS_HOME/watch_config.yml\nRuntime files: GITNEXUS_HOME/watch/watch.pid, watch.mutex, watch.owner.json, watch.status.json, auto-sync-state.json\nRecovery: mutexes with verified dead owners are reclaimed automatically; invalid or legacy mutexes fail closed and require manual removal after confirming no watch process is running.\nWrites: GITNEXUS_HOME/watch/project_commit_info.txt\nRemote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.\nRuns once immediately, then repeats on sync_interval_minutes.', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 04b319d25..7ecb4dc72 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -129,6 +129,8 @@ export const zhCN = { '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', 'help.command.watch.description': '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', + 'help.watch.details': + '\n操作：init、start（默认）、restart、stop、status、reset\n配置：GITNEXUS_HOME/watch_config.yml\n运行时文件：GITNEXUS_HOME/watch/watch.pid、watch.mutex、watch.owner.json、watch.status.json、auto-sync-state.json\n恢复：已验证 owner 退出的 mutex 会自动回收；无效或旧版 mutex 会安全拒绝，确认没有 watch 进程运行后再手动删除。\n写入：GITNEXUS_HOME/watch/project_commit_info.txt\n远程地址：仅允许 git@github.com:owner/repo.git、git@gitlab.com:group/repo.git 和 git@gitee.com:owner/repo.git。\n启动后立即运行一次，之后按 sync_interval_minutes 重复。', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 4dc6decf6..182b39c7b 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -50,18 +50,7 @@ program .description( 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', ) - .addHelpText( - 'after', - [ - '', - 'Actions: init, start (default), restart, stop, status', - 'Configuration: GITNEXUS_HOME/watch_config.yml', - 'Runtime files: GITNEXUS_HOME/watch/watch.pid, watch.lock, watch.status.json, auto-sync-state.json', - 'Writes: GITNEXUS_HOME/watch/project_commit_info.txt', - 'Remote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.', - 'Runs once immediately, then repeats on sync_interval_minutes.', - ].join('\n'), - ) + .addHelpText('after', () => t('help.watch.details')) .action(createLazyAction(() => import('./watch.js'), 'watchCommand')); // Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts index 41dc2ba59..ed7d4e0a9 100644 --- a/gitnexus/src/cli/watch.ts +++ b/gitnexus/src/cli/watch.ts @@ -2,7 +2,9 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { getAutoSyncConfigPath, + getAutoSyncMutexPath, readAutoSyncWatchStatus, + resetAutoSyncState, startAutoSyncWatch, stopAutoSyncWatch, type WatchStatusRecord, @@ -13,17 +15,28 @@ export async function watchCommand(action = 'start'): Promise { await initWatchConfig(); return; } + if (action === 'reset') { + if (!(await resetAutoSyncState())) { + process.stderr.write( + `[auto-sync] Cannot reset analysis state while the watch mutex is held. Confirm no watch process is running, then remove ${getAutoSyncMutexPath()}.\n`, + ); + process.exitCode = 1; + return; + } + process.stdout.write('[auto-sync] Reset analysis state.\n'); + return; + } if (action === 'status') { printStatus(await readAutoSyncWatchStatus()); return; } if (action === 'stop') { - await stopAutoSyncWatch(); + if ((await stopAutoSyncWatch()) !== 'stopped') process.exitCode = 1; return; } if (action === 'restart') { - const stopped = await stopAutoSyncWatch(); - if (!stopped) { + const result = await stopAutoSyncWatch(); + if (result === 'refused' || result === 'timeout') { process.exitCode = 1; return; } @@ -46,10 +59,17 @@ async function startWatchProcess(): Promise { } const stop = () => { - void handle.stop().finally(() => { - process.stderr.write('[auto-sync] Watch stopped.\n'); - process.exit(0); - }); + void handle.stop().then( + () => { + process.stderr.write('[auto-sync] Watch stopped.\n'); + process.exit(0); + }, + (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[auto-sync] Failed to stop watch: ${message}\n`); + process.exit(1); + }, + ); }; process.once('SIGINT', stop); process.once('SIGTERM', stop); @@ -70,7 +90,7 @@ async function initWatchConfig(): Promise { await fs.mkdir(path.dirname(configPath), { recursive: true }); await fs.writeFile( configPath, - defaultSyncConfig(path.resolve(path.dirname(configPath), 'repo')), + defaultSyncConfig(path.resolve(path.dirname(configPath), 'repos')), { flag: 'wx', }, @@ -96,7 +116,6 @@ function defaultSyncConfig(localPath: string): string { 'projects:', ` - local_path: ${localPath}`, ' branches: [master, main]', - ' group_name: back_end', ' overwrite_local_changes: false', ' remote_urls:', ' - git@github.com:owner/repo.git', diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 8b8d492b2..cda91902a 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -16,7 +16,10 @@ export type AutoSyncAnalysisRunner = ( signal?: AbortSignal, ) => Promise>; -interface AnalysisWorker extends Pick {} +interface AnalysisWorker extends Pick { + stdout?: Pick | null; + stderr?: Pick | null; +} export interface AutoSyncAnalysisLaunchDeps { forkWorker: (workerPath: string, execArgv: string[]) => AnalysisWorker; @@ -59,61 +62,73 @@ export function createAutoSyncAnalysisRunner( ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href, '--max-old-space-size=8192'] : ['--max-old-space-size=8192']; const child = deps.forkWorker(workerPath, execArgv); - let outcome: WorkerMessage | undefined; - let timedOut = false; - let cancelled = false; + child.stdout?.resume(); + child.stderr?.resume(); + + let terminalOutcome: WorkerMessage | undefined; let terminationGrace: ReturnType | undefined; - const timeout = deps.setTimeoutFn(() => { - timedOut = true; - child.kill('SIGTERM'); - terminationGrace = deps.setTimeoutFn(() => child.kill('SIGKILL'), TERMINATION_GRACE_MS); - }, timeoutMs); - const onAbort = () => { - cancelled = true; + let settled = false; + const cleanup = () => { deps.clearTimeoutFn(timeout); if (terminationGrace) deps.clearTimeoutFn(terminationGrace); + signal?.removeEventListener('abort', onAbort); + }; + const settle = (error?: Error, result?: Pick) => { + if (settled) return; + settled = true; + cleanup(); + if (error) reject(error); + else resolve(result!); + }; + const timeout = deps.setTimeoutFn(() => { + child.kill('SIGTERM'); + terminationGrace = deps.setTimeoutFn(() => { + child.kill('SIGKILL'); + settle(new Error(`Analysis timed out after ${timeoutMs}ms.`)); + }, TERMINATION_GRACE_MS); + }, timeoutMs); + const onAbort = () => { child.kill('SIGKILL'); + settle(new Error('Analysis cancelled.')); }; signal?.addEventListener('abort', onAbort, { once: true }); child.on('message', (message: WorkerMessage) => { - if (message.type !== 'progress') outcome ??= message; - else outcome = message; + if (message.type !== 'progress') terminalOutcome ??= message; }); child.on('error', (error) => { - outcome = { type: 'error', message: `Auto-sync analyze worker error: ${error.message}` }; + settle(new Error(`Auto-sync analyze worker error: ${error.message}`)); }); child.on('exit', (code, childSignal) => { - deps.clearTimeoutFn(timeout); - if (terminationGrace) deps.clearTimeoutFn(terminationGrace); - signal?.removeEventListener('abort', onAbort); - if (cancelled) { - reject(new Error('Analysis cancelled.')); - return; - } - if (timedOut) { - reject( + if (settled) return; + if (terminationGrace) { + settle( new Error( `Analysis timed out after ${timeoutMs}ms and worker exited (${childSignal ?? code ?? 'unknown'}).`, ), ); return; } - if (outcome?.type === 'complete') { - resolve({ stats: outcome.result.stats }); + if (terminalOutcome?.type === 'complete') { + settle(undefined, { stats: terminalOutcome.result.stats }); return; } - if (outcome?.type === 'error') { - reject(new Error(outcome.message)); + if (terminalOutcome?.type === 'error') { + settle(new Error(terminalOutcome.message)); return; } - reject( + settle( new Error( - `Auto-sync analyze worker exited before completion (${signal ?? code ?? 'unknown'}).`, + `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, ), ); }); - child.send({ type: 'start', repoPath, options }); + try { + child.send({ type: 'start', repoPath, options }); + } catch (error) { + child.kill('SIGKILL'); + settle(new Error(`Failed to start auto-sync analyze worker: ${(error as Error).message}`)); + } }); } diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 7096b1cbc..f85c0b812 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -120,6 +120,8 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy : parseDurationMs(raw.repo_git_timeout); if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) { errors.push('repo_git_timeout must be a positive duration such as 10s'); + } else if (repoGitTimeoutMs > MAX_TIMER_DELAY_MS) { + errors.push(`repo_git_timeout must not exceed ${MAX_TIMER_DELAY_MS}ms`); } const maxAnalyzeTimeoutMs = diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts index 4182843bf..b02948779 100644 --- a/gitnexus/src/core/auto-sync/index.ts +++ b/gitnexus/src/core/auto-sync/index.ts @@ -13,10 +13,12 @@ export { } from './config.js'; export { buildStateKey, + getAutoSyncMutexPath, getAutoSyncWatchDir, getAutoSyncStatePath, getProjectCommitInfoPath, loadAutoSyncState, + resetAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, writeProjectCommitInfo, @@ -49,6 +51,7 @@ export { startAutoSyncWatch, stopAutoSyncWatch, type AutoSyncStartHandle, + type AutoSyncWatchStopResult, type AutoSyncWatchPaths, type WatchStatusRecord, } from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts index 7359c4679..2b1d3e77b 100644 --- a/gitnexus/src/core/auto-sync/path-security.ts +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -4,6 +4,16 @@ import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; import { getAutoSyncWatchDir } from './state.js'; +const WINDOWS_DANGEROUS_ROOTS = + process.platform === 'win32' + ? [ + process.env.SystemRoot, + process.env.ProgramData, + process.env.ProgramFiles, + process.env['ProgramFiles(x86)'], + ].filter((entry): entry is string => Boolean(entry)) + : []; + const DANGEROUS_ROOTS = new Set( [ '/', @@ -25,6 +35,7 @@ const DANGEROUS_ROOTS = new Set( '/tmp', '/usr', '/var', + ...WINDOWS_DANGEROUS_ROOTS, ].map((entry) => path.resolve(entry)), ); @@ -47,6 +58,7 @@ const DANGEROUS_PARENT_ROOTS = new Set( '/tmp', '/usr', '/var', + ...WINDOWS_DANGEROUS_ROOTS, ].map((entry) => path.resolve(entry)), ); @@ -72,10 +84,12 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise { + const cutoff = Date.now() - QUARANTINE_RETENTION_DAYS * 24 * 60 * 60 * 1_000; + let entries; + try { + entries = await fs.readdir(quarantineRoot); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return; + throw err; + } + await Promise.all( + entries + .filter((entry) => entry.startsWith('auto-sync-')) + .map(async (entry) => { + const entryPath = path.join(quarantineRoot, entry); + const stat = await fs.stat(entryPath).catch(() => undefined); + if (stat && stat.mtimeMs < cutoff) { + await fs.rm(entryPath, { recursive: true, force: true }); + } + }), + ); +} + function assertNotDangerousRoot(root: string): void { + if (root === path.resolve(getGlobalDir(), 'repos')) return; if (DANGEROUS_ROOTS.has(root)) throw new Error(`Refusing unsafe auto-sync clone root: ${root}`); for (const dangerousRoot of DANGEROUS_PARENT_ROOTS) { const rel = path.relative(dangerousRoot, root); @@ -167,14 +210,20 @@ async function assertNoSymlinkPath(root: string): Promise { export async function assertDirectoryOwnerAndPermissions(root: string): Promise { const stat = await fs.stat(root); if (!stat.isDirectory()) throw new Error(`auto-sync clone root is not a directory: ${root}`); + if (process.platform === 'win32') { + throw new Error('auto-sync clone root ownership/ACL verification is not supported on Windows'); + } if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { throw new Error(`auto-sync clone root is owned by uid ${stat.uid}, not current process uid`); } const mode = stat.mode & 0o777; + const groupWritable = (mode & 0o020) !== 0; const worldWritable = (mode & 0o002) !== 0; - const sticky = (stat.mode & 0o1000) !== 0; - if (worldWritable && !sticky) { - throw new Error(`Refusing world-writable auto-sync clone root without sticky bit: ${root}`); + if (worldWritable) { + throw new Error(`Refusing world-writable auto-sync clone root: ${root}`); + } + if (groupWritable) { + throw new Error(`Refusing group-writable auto-sync clone root: ${root}`); } } diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index 9f18d39a3..70d159eba 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -1,14 +1,12 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import yaml from 'js-yaml'; +import { createRequire } from 'node:module'; import { loadGroupConfig } from '../group/config-parser.js'; import { getDefaultGitnexusDir, getGroupDir } from '../group/storage.js'; import { syncGroup } from '../group/sync.js'; -import { runFullAnalysis } from '../run-analyze.js'; -import { getCurrentBranch, getCurrentCommit } from '../../storage/git.js'; import { registerRepo, type RepoMeta } from '../../storage/repo-manager.js'; import { extractRepoNameFromRemoteUrl } from './repo.js'; -import { cloneOrPull } from '../../server/git-clone.js'; +import { cloneOrPull, runGit } from '../../server/git-clone.js'; import { resolveConfiguredCloneRoot } from './path-security.js'; import { buildStateKey, @@ -32,9 +30,8 @@ export interface AutoSyncLogger { export interface AutoSyncRunDeps { cloneOrPull: typeof cloneOrPull; - getCurrentBranch: typeof getCurrentBranch; - getCurrentCommit: typeof getCurrentCommit; - runFullAnalysis?: typeof runFullAnalysis; + getCurrentBranch: (repoPath: string, timeoutMs: number) => Promise; + getCurrentCommit: (repoPath: string, timeoutMs: number) => Promise; runAnalysis: AutoSyncAnalysisRunner; registerRepo: typeof registerRepo; loadState: typeof loadAutoSyncState; @@ -53,6 +50,9 @@ export interface AutoSyncRunResult { failed: number; } +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + const DEFAULT_LOGGER: AutoSyncLogger = { info: (message) => process.stderr.write(`${message}\n`), warn: (message) => process.stderr.write(`${message}\n`), @@ -61,8 +61,12 @@ const DEFAULT_LOGGER: AutoSyncLogger = { const DEFAULT_DEPS: AutoSyncRunDeps = { cloneOrPull, - getCurrentBranch, - getCurrentCommit, + getCurrentBranch: async (repoPath, timeoutMs) => { + const branch = (await runGit(['branch', '--show-current'], repoPath, { timeoutMs })).trim(); + return branch || undefined; + }, + getCurrentCommit: async (repoPath, timeoutMs) => + (await runGit(['rev-parse', 'HEAD'], repoPath, { timeoutMs })).trim(), runAnalysis: runAutoSyncAnalysis, registerRepo, loadState: loadAutoSyncState, @@ -109,9 +113,11 @@ export async function runAutoSyncOnce( const lastSyncTime = now().toISOString(); try { throwIfAborted(options.signal); - validateAutoSyncRemoteUrl(item.remoteUrl); - const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl); - const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName); + if (!item.cloneRoot || !item.repoName || !item.targetDir) { + throw new Error(item.error ?? 'Invalid auto-sync work item'); + } + const repoName = item.repoName; + const targetDir = item.targetDir; const syncResult = await syncFirstAvailableBranch({ item, repoName, @@ -139,7 +145,7 @@ export async function runAutoSyncOnce( const currentBranch = syncResult.branch; - const currentCommit = deps.getCurrentCommit(targetDir); + const currentCommit = await deps.getCurrentCommit(targetDir, config.repoGitTimeoutMs); const stateKey = buildStateKey(targetDir, currentBranch); const previous = state[stateKey]; let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; @@ -166,18 +172,12 @@ export async function runAutoSyncOnce( }) ) { try { - const analysis = deps.runFullAnalysis - ? await deps.runFullAnalysis( - targetDir, - { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, - { onProgress: () => {} }, - ) - : await deps.runAnalysis( - targetDir, - { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, - config.analyzeTimeoutMs, - options.signal, - ); + const analysis = await deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + ); throwIfAborted(options.signal); stats = analysis.stats; analyzeStatus = 'success'; @@ -245,16 +245,11 @@ export async function runAutoSyncOnce( } result.synced += 1; - const stateEntry: AutoSyncCommitStateEntry = { - codeCommitId: repoResult.currentCommit, - analyzedCommitId: repoResult.analyzedCommitId, - lastAnalyzeStatus: repoResult.analyzeStatus, - analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, - lastAnalyzeError: repoResult.lastAnalyzeError, - lastSyncTime: repoResult.lastSyncTime, - }; - state[repoResult.stateKey] = stateEntry; - if (repoResult.analyzeStatus === 'success') { + let analyzeStatus = repoResult.analyzeStatus; + let analyzeConsecutiveFailures = repoResult.analyzeConsecutiveFailures; + let lastAnalyzeError = repoResult.lastAnalyzeError; + let analyzedCommitId = repoResult.analyzedCommitId; + if (analyzeStatus === 'success') { const meta: RepoMeta = { repoPath: repoResult.targetDir, lastCommit: repoResult.currentCommit, @@ -262,28 +257,45 @@ export async function runAutoSyncOnce( stats: repoResult.stats!, branch: repoResult.branch, }; - await deps.registerRepo(repoResult.targetDir, meta, { - name: getAutoSyncRepoIdentity(repoResult.remoteUrl), - }); - result.analyzed += 1; - } else if (repoResult.analyzeStatus === 'failed') { + try { + await deps.registerRepo(repoResult.targetDir, meta, { + name: getAutoSyncRepoIdentity(repoResult.remoteUrl), + }); + result.analyzed += 1; + } catch (err: unknown) { + analyzeStatus = 'failed'; + analyzedCommitId = undefined; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = `Repository registration failed: ${shortErrorMessage(err)}`; + result.failed += 1; + logger.error(`[auto-sync] ${lastAnalyzeError}`); + } + } else if (analyzeStatus === 'failed') { result.failed += 1; - } else if (repoResult.analyzeStatus === 'threshold_skipped') { - result.skippedAnalysis += 1; } else { result.skippedAnalysis += 1; } + const stateEntry: AutoSyncCommitStateEntry = { + codeCommitId: repoResult.currentCommit, + analyzedCommitId, + lastAnalyzeStatus: analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }; + state[repoResult.stateKey] = stateEntry; + commitInfoEntries.push({ remoteUrl: repoResult.remoteUrl, localPath: repoResult.targetDir, branch: repoResult.branch, codeCommitId: repoResult.currentCommit, - analyzedCommitId: repoResult.analyzedCommitId, - status: repoResult.analyzeStatus, - analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, + analyzedCommitId, + status: analyzeStatus, + analyzeConsecutiveFailures, analyzeFailureThreshold: config.analyzeFailureThreshold, - lastAnalyzeError: repoResult.lastAnalyzeError, + lastAnalyzeError, lastSyncTime: repoResult.lastSyncTime, }); @@ -302,7 +314,7 @@ export async function runAutoSyncOnce( `[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`, ); } - if (groupMembershipOk && repoResult.analyzeStatus === 'success') { + if (groupMembershipOk && analyzeStatus === 'success') { groupsToSync.add(repoResult.project.groupName); } } @@ -329,8 +341,11 @@ function shortErrorMessage(err: unknown): string { export function getConfiguredRepoPath( project: Pick, repoName: string, + remoteUrl?: string, ): string { - return path.resolve(project.localPath, repoName); + if (!remoteUrl) return path.resolve(project.localPath, repoName); + const identity = getAutoSyncRepoIdentity(remoteUrl); + return path.resolve(project.localPath, ...identity.split('/').slice(0, -1), repoName); } export async function addRepoToGroup( @@ -380,11 +395,19 @@ async function buildWorkItems( const items: AutoSyncWorkItem[] = []; const targetOwners = new Map(); for (const project of config.projects) { - const cloneRoot = await deps.resolveCloneRoot(project.localPath); + let cloneRoot: AutoSyncWorkItem['cloneRoot']; + try { + cloneRoot = await deps.resolveCloneRoot(project.localPath); + } catch (err: unknown) { + for (const remoteUrl of project.remoteUrls) { + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); + } + continue; + } for (const remoteUrl of project.remoteUrls) { try { const repoName = extractRepoNameFromRemoteUrl(remoteUrl); - const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName, remoteUrl); const previous = targetOwners.get(targetDir); if (previous !== undefined) { throw new Error( @@ -392,10 +415,10 @@ async function buildWorkItems( ); } targetOwners.set(targetDir, remoteUrl); + items.push({ project, remoteUrl, cloneRoot, repoName, targetDir }); } catch (err: unknown) { - if ((err as Error).message.startsWith('Duplicate auto-sync targetDir')) throw err; + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); } - items.push({ project, remoteUrl, cloneRoot }); } } return items; @@ -429,7 +452,10 @@ function throwIfAborted(signal: AbortSignal | undefined): void { interface AutoSyncWorkItem { project: AutoSyncProjectConfig; remoteUrl: string; - cloneRoot: Awaited>; + cloneRoot?: Awaited>; + repoName?: string; + targetDir?: string; + error?: string; } async function syncFirstAvailableBranch(input: { @@ -448,15 +474,15 @@ async function syncFirstAvailableBranch(input: { for (const branch of input.item.project.branches) { try { await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, { - allowedCloneRoot: input.item.cloneRoot.root, + allowedCloneRoot: input.item.cloneRoot!.root, expectedRepoName: input.repoName, - quarantineRoot: input.item.cloneRoot.quarantineRoot, + quarantineRoot: input.item.cloneRoot!.quarantineRoot, allowAutoSyncSsh: true, timeoutMs: input.timeoutMs, branch, overwriteLocalChanges: input.item.project.overwriteLocalChanges, }); - const currentBranch = input.deps.getCurrentBranch(input.targetDir); + const currentBranch = await input.deps.getCurrentBranch(input.targetDir, input.timeoutMs); if (currentBranch === branch) return { ok: true, branch }; failures.push(`${branch}: checked out ${currentBranch ?? ''}`); input.logger.warn( diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index bad0c901b..026ea512b 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -2,16 +2,19 @@ import fs from 'node:fs/promises'; import crypto from 'node:crypto'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; import { getGlobalDir } from '../../storage/repo-manager.js'; +import { isProcessAlive, readProcessStartTime } from '../../utils/process-identity.js'; import { loadAutoSyncConfig } from './config.js'; import { runAutoSyncOnce } from './runner.js'; -import { getAutoSyncWatchDir } from './state.js'; +import { getAutoSyncMutexPath, getAutoSyncWatchDir } from './state.js'; export interface AutoSyncStartHandle { stop(): Promise; } export type WatchStatusState = 'running' | 'stopping' | 'stopped' | 'stale' | 'error'; +export type AutoSyncWatchStopResult = 'stopped' | 'not_running' | 'refused' | 'timeout'; export interface WatchStatusRecord { state: WatchStatusState; @@ -22,21 +25,24 @@ export interface WatchStatusRecord { updatedAt: string; } -export interface WatchLockRecord { +export interface WatchOwnerRecord { pid: number; ownerId: string; + processStartTime: string; createdAt: string; } export interface AutoSyncWatchPaths { pidPath: string; - lockPath: string; + mutexPath: string; + ownerPath: string; statusPath: string; } export interface AutoSyncWatchControlDeps { isProcessAlive(pid: number): boolean; readProcessCommand(pid: number): string | undefined; + readProcessStartTime(pid: number): string | undefined; killProcess(pid: number, signal?: NodeJS.Signals): void; sleep(ms: number): Promise; } @@ -45,7 +51,8 @@ export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWat const watchDir = getAutoSyncWatchDir(gitnexusDir); return { pidPath: path.join(watchDir, 'watch.pid'), - lockPath: path.join(watchDir, 'watch.lock'), + mutexPath: getAutoSyncMutexPath(gitnexusDir), + ownerPath: path.join(watchDir, 'watch.owner.json'), statusPath: path.join(watchDir, 'watch.status.json'), }; } @@ -65,169 +72,152 @@ export async function startAutoSyncWatch( const paths = options.paths ?? getAutoSyncWatchPaths(); const deps = resolveWatchDeps(options.deps); const ownerId = crypto.randomUUID(); - await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); - const lockHandle = await acquireWatchLock(paths, deps, stderr); - if (!lockHandle) return null; - await lockHandle.writeFile( - `${JSON.stringify({ pid: process.pid, ownerId, createdAt: new Date().toISOString() })}\n`, - 'utf-8', - ); - await fs.writeFile(paths.pidPath, `${process.pid}\n`, 'utf-8'); - - const loaded = await loadAutoSyncConfig(); - if (loaded.ok === false) { - stderr.write(`${loaded.message}\n`); - await writeWatchStatus(paths, { - state: 'error', - pid: process.pid, - ownerId, - message: loaded.message, - updatedAt: new Date().toISOString(), - }); - await cleanupWatchFiles(paths, lockHandle); + const processStartTime = deps.readProcessStartTime(process.pid); + if (!processStartTime) { + stderr.write('[auto-sync] Unable to verify the watch process start time.\n'); return null; } - await writeWatchStatus(paths, { - state: 'running', - pid: process.pid, - ownerId, - configPath: loaded.config.configPath, - updatedAt: new Date().toISOString(), - }); + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + const releaseLock = await acquireWatchLock(paths, deps, stderr, processStartTime); + if (!releaseLock) return null; - const runOnce = options.runOnce ?? runAutoSyncOnce; - let activeRun: Promise | undefined; - let activeAbortController: AbortController | undefined; - const runSafely = () => { - if (activeRun) { - stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); - return; - } - const startedAt = new Date(); - stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); - const abortController = new AbortController(); - const run = runOnce(loaded.config, { signal: abortController.signal }) - .then((result) => { - stderr.write( - `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, - ); - }) - .catch((err: unknown) => { - stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); - stderr.write('[auto-sync] Watch loop finished: failed.\n'); - }); - activeRun = run; - activeAbortController = abortController; - void run.finally(() => { - if (activeRun === run) { - activeRun = undefined; - activeAbortController = undefined; - } + try { + await writeWatchOwner(paths, { + pid: process.pid, + ownerId, + processStartTime, + createdAt: new Date().toISOString(), }); - }; + await writeAtomicText(paths.pidPath, `${process.pid}\n`); - runSafely(); - const intervalMs = loaded.config.syncIntervalMinutes * 60_000; - const setIntervalFn = options.setIntervalFn ?? setInterval; - const clearIntervalFn = options.clearIntervalFn ?? clearInterval; - const timer = setIntervalFn(runSafely, intervalMs); - if (options.keepAlive === false) timer.unref?.(); - return { - stop: async () => { - clearIntervalFn(timer); - activeAbortController?.abort(); + const loaded = await loadAutoSyncConfig(); + if (loaded.ok === false) { + stderr.write(`${loaded.message}\n`); await writeWatchStatus(paths, { - state: 'stopping', + state: 'error', pid: process.pid, ownerId, - configPath: loaded.config.configPath, + message: loaded.message, updatedAt: new Date().toISOString(), }); - await activeRun?.catch(() => {}); - await writeWatchStatus(paths, { - state: 'stopped', - pid: process.pid, - ownerId, - configPath: loaded.config.configPath, - updatedAt: new Date().toISOString(), - }).finally(() => cleanupWatchFiles(paths, lockHandle)); - }, - }; + await cleanupWatchFiles(paths, ownerId, releaseLock); + return null; + } + await writeWatchStatus(paths, { + state: 'running', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + + const runOnce = options.runOnce ?? runAutoSyncOnce; + let activeRun: Promise | undefined; + let activeAbortController: AbortController | undefined; + const runSafely = () => { + if (activeRun) { + stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); + return; + } + const startedAt = new Date(); + stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); + const abortController = new AbortController(); + const run = runOnce(loaded.config, { signal: abortController.signal }) + .then((result) => { + stderr.write( + `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, + ); + }) + .catch((err: unknown) => { + stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + stderr.write('[auto-sync] Watch loop finished: failed.\n'); + }); + activeRun = run; + activeAbortController = abortController; + void run.finally(() => { + if (activeRun === run) { + activeRun = undefined; + activeAbortController = undefined; + } + }); + }; + + runSafely(); + const intervalMs = loaded.config.syncIntervalMinutes * 60_000; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; + const timer = setIntervalFn(runSafely, intervalMs); + if (options.keepAlive === false) timer.unref?.(); + let stopPromise: Promise | undefined; + return { + stop: () => + (stopPromise ??= (async () => { + clearIntervalFn(timer); + activeAbortController?.abort(); + try { + await writeWatchStatus(paths, { + state: 'stopping', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + await activeRun?.catch(() => {}); + await writeWatchStatus(paths, { + state: 'stopped', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + } finally { + await cleanupWatchFiles(paths, ownerId, releaseLock); + } + })()), + }; + } catch (error) { + await cleanupWatchFiles(paths, ownerId, releaseLock).catch(() => {}); + throw error; + } } async function acquireWatchLock( paths: AutoSyncWatchPaths, deps: AutoSyncWatchControlDeps, stderr: Pick, -): Promise { + processStartTime: string, +): Promise<(() => Promise) | null> { try { - return await fs.open(paths.lockPath, 'wx'); + return await acquireFileLock(paths.mutexPath, { + pid: process.pid, + processStartTime, + isProcessAlive: deps.isProcessAlive, + readProcessStartTime: deps.readProcessStartTime, + }); } catch (err: unknown) { - if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + if (!(err instanceof FileLockBusyError)) throw err; } - const lock = await readLockFile(paths.lockPath); - if (!lock) { - const message = 'watch lock already exists but has no readable owner; refusing to start'; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'error', - message, - updatedAt: new Date().toISOString(), - }); + const owner = await readOwnerFile(paths.ownerPath); + if (!owner) { + stderr.write( + `[auto-sync] Watch mutex is held but owner metadata is not ready or invalid. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); return null; } - - if (deps.isProcessAlive(lock.pid)) { - const reason = getWatchProcessIdentityError(lock.pid, deps); - if (reason) { - stderr.write(`[auto-sync] Refusing to trust existing watch pid ${lock.pid}; ${reason}.\n`); - await writeWatchStatus(paths, { - state: 'error', - pid: lock.pid, - ownerId: lock.ownerId, - message: reason, - updatedAt: new Date().toISOString(), - }); - return null; - } - stderr.write(`[auto-sync] Watch is already running with pid ${lock.pid}.\n`); - await writeWatchStatus(paths, { - state: 'running', - pid: lock.pid, - ownerId: lock.ownerId, - message: 'watch already running', - updatedAt: new Date().toISOString(), - }); + if (!deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch mutex remains after owner pid ${owner.pid} exited. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); return null; } - - stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); - await removeIfExists(paths.pidPath); - await removeIfExists(paths.lockPath); - await writeWatchStatus(paths, { - state: 'stale', - pid: lock.pid, - ownerId: lock.ownerId, - message: 'removed stale lock and pid', - updatedAt: new Date().toISOString(), - }); - - try { - return await fs.open(paths.lockPath, 'wx'); - } catch (err: unknown) { - if ((err as NodeJS.ErrnoException).code === 'EEXIST') { - const message = 'watch lock was reacquired by another process; refusing to start'; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'error', - message, - updatedAt: new Date().toISOString(), - }); - return null; - } - throw err; + const reason = getWatchProcessIdentityError(owner, deps); + if (reason) { + stderr.write(`[auto-sync] Refusing to trust existing watch pid ${owner.pid}; ${reason}.\n`); + return null; } + stderr.write(`[auto-sync] Watch is already running with pid ${owner.pid}.\n`); + return null; } export async function stopAutoSyncWatch( @@ -238,7 +228,7 @@ export async function stopAutoSyncWatch( timeoutMs?: number; pollMs?: number; } = {}, -): Promise { +): Promise { const stderr = options.stderr ?? process.stderr; const paths = options.paths ?? getAutoSyncWatchPaths(); const deps = resolveWatchDeps(options.deps); @@ -246,105 +236,54 @@ export async function stopAutoSyncWatch( const pollMs = options.pollMs ?? 100; const pid = await readPid(paths.pidPath); if (!pid) { - const lock = await readLockFile(paths.lockPath); - if (lock && deps.isProcessAlive(lock.pid)) { - const message = `watch appears to be starting with pid ${lock.pid}; pid file is not ready`; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'error', - pid: lock.pid, - ownerId: lock.ownerId, - message, - updatedAt: new Date().toISOString(), - }); - return false; + const owner = await readOwnerFile(paths.ownerPath); + if (owner && deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch appears to be starting with pid ${owner.pid}; pid file is not ready.\n`, + ); + return 'refused'; } - if (lock) { - stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); - await removeIfExists(paths.lockPath); - await writeWatchStatus(paths, { - state: 'stale', - pid: lock.pid, - ownerId: lock.ownerId, - message: 'removed stale lock without pid file', - updatedAt: new Date().toISOString(), - }); - return false; - } - if (await fileExists(paths.lockPath)) { - const message = 'watch lock exists but has no readable owner; refusing to stop'; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'error', - message, - updatedAt: new Date().toISOString(), - }); - return false; + if (owner || (await fileExists(paths.mutexPath))) { + stderr.write( + `[auto-sync] Watch ownership is stale or incomplete. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; } stderr.write('[auto-sync] Watch is not running.\n'); - await writeWatchStatus(paths, { - state: 'stopped', - message: 'no pid file', - updatedAt: new Date().toISOString(), - }); - return false; + return 'not_running'; } if (!deps.isProcessAlive(pid)) { - stderr.write(`[auto-sync] Removing stale watch pid ${pid}.\n`); - await removeIfExists(paths.pidPath); - await removeIfExists(paths.lockPath); - await writeWatchStatus(paths, { - state: 'stale', - pid, - message: 'removed stale pid and lock', - updatedAt: new Date().toISOString(), - }); - return false; + stderr.write( + `[auto-sync] Watch pid ${pid} is stale. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; } + const owner = await readVerifiedWatchOwner(paths, pid, deps); if (owner.ok === false) { - const message = `refusing to stop pid ${pid}; ${owner.reason}`; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'error', - pid, - message, - updatedAt: new Date().toISOString(), - }); - return false; + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; ${owner.reason}.\n`); + return 'refused'; } - await writeWatchStatus(paths, { - state: 'stopping', - pid, - ownerId: owner.owner.ownerId, - message: 'stop signal sent; waiting for watch process to exit', - updatedAt: new Date().toISOString(), - }); + + const currentPid = await readPid(paths.pidPath); + const currentOwner = await readVerifiedWatchOwner(paths, pid, deps); + if ( + currentPid !== pid || + currentOwner.ok === false || + currentOwner.owner.ownerId !== owner.owner.ownerId + ) { + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; watch ownership changed.\n`); + return 'refused'; + } + deps.killProcess(pid, 'SIGTERM'); stderr.write(`[auto-sync] Stop signal sent to watch pid ${pid}.\n`); const stopped = await waitForProcessExit(pid, { deps, timeoutMs, pollMs }); if (!stopped) { - const message = `watch pid ${pid} did not exit within ${timeoutMs}ms`; - stderr.write(`[auto-sync] ${message}.\n`); - await writeWatchStatus(paths, { - state: 'stopping', - pid, - ownerId: owner.owner.ownerId, - message, - updatedAt: new Date().toISOString(), - }); - return false; + stderr.write(`[auto-sync] Watch pid ${pid} did not exit within ${timeoutMs}ms.\n`); + return 'timeout'; } - await removeIfExists(paths.pidPath); - await removeIfExists(paths.lockPath); - await writeWatchStatus(paths, { - state: 'stopped', - pid, - ownerId: owner.owner.ownerId, - message: 'watch stopped', - updatedAt: new Date().toISOString(), - }); - return true; + return 'stopped'; } export async function readAutoSyncWatchStatus( @@ -388,17 +327,19 @@ export async function readAutoSyncWatchStatus( return stored ?? { state: 'stopped', updatedAt: new Date().toISOString() }; } -async function readLockFile(lockPath: string): Promise { +async function readOwnerFile(ownerPath: string): Promise { try { - const raw = await fs.readFile(lockPath, 'utf-8'); - const parsed = JSON.parse(raw) as WatchLockRecord; + const raw = await fs.readFile(ownerPath, 'utf-8'); + const parsed = JSON.parse(raw) as WatchOwnerRecord; if ( parsed && typeof parsed === 'object' && Number.isInteger(parsed.pid) && parsed.pid > 0 && typeof parsed.ownerId === 'string' && - parsed.ownerId + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime ) { return parsed; } @@ -413,28 +354,31 @@ async function readVerifiedWatchOwner( paths: AutoSyncWatchPaths, pid: number, deps: AutoSyncWatchControlDeps, -): Promise<{ ok: true; owner: WatchLockRecord } | { ok: false; reason: string }> { - const [status, lock] = await Promise.all([ +): Promise<{ ok: true; owner: WatchOwnerRecord } | { ok: false; reason: string }> { + const [status, owner] = await Promise.all([ readStatusFile(paths.statusPath), - readLockFile(paths.lockPath), + readOwnerFile(paths.ownerPath), ]); - if (!lock) return { ok: false, reason: 'watch lock is missing or invalid' }; + if (!owner) return { ok: false, reason: 'watch owner is missing or invalid' }; if (!status) return { ok: false, reason: 'watch status is missing or invalid' }; - if (lock.pid !== pid) return { ok: false, reason: 'watch lock pid does not match pid file' }; + if (owner.pid !== pid) return { ok: false, reason: 'watch owner pid does not match pid file' }; if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' }; - if (!status.ownerId || status.ownerId !== lock.ownerId) { - return { ok: false, reason: 'watch status owner does not match lock owner' }; + if (!status.ownerId || status.ownerId !== owner.ownerId) { + return { ok: false, reason: 'watch status owner does not match watch owner' }; } - const identityError = getWatchProcessIdentityError(pid, deps); + const identityError = getWatchProcessIdentityError(owner, deps); if (identityError) return { ok: false, reason: identityError }; - return { ok: true, owner: lock }; + return { ok: true, owner }; } function getWatchProcessIdentityError( - pid: number, + owner: WatchOwnerRecord, deps: AutoSyncWatchControlDeps, ): string | undefined { - const command = deps.readProcessCommand(pid); + const processStartTime = deps.readProcessStartTime(owner.pid); + if (!processStartTime) return 'unable to verify process start time'; + if (processStartTime !== owner.processStartTime) return 'pid belongs to a different process'; + const command = deps.readProcessCommand(owner.pid); if (!command) return 'unable to verify process command'; if ( !/(?:^|\s)watch(?:\s|$)/.test(command) || @@ -492,13 +436,33 @@ async function writeWatchStatus( await fs.rename(tmpPath, paths.statusPath); } +async function writeWatchOwner(paths: AutoSyncWatchPaths, record: WatchOwnerRecord): Promise { + await writeAtomicText(paths.ownerPath, `${JSON.stringify(record, null, 2)}\n`); +} + async function cleanupWatchFiles( paths: AutoSyncWatchPaths, - lockHandle?: fs.FileHandle, + ownerId: string, + releaseLock: () => Promise, ): Promise { - await lockHandle?.close().catch(() => {}); - await removeIfExists(paths.pidPath); - await removeIfExists(paths.lockPath); + try { + const owner = await readOwnerFile(paths.ownerPath); + if (owner?.ownerId === ownerId) { + if ((await readPid(paths.pidPath)) === owner.pid) await removeIfExists(paths.pidPath); + if ((await readOwnerFile(paths.ownerPath))?.ownerId === ownerId) { + await removeIfExists(paths.ownerPath); + } + } + } finally { + await releaseLock(); + } +} + +async function writeAtomicText(filePath: string, content: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, content, 'utf-8'); + await fs.rename(tmpPath, filePath); } async function removeIfExists(filePath: string): Promise { @@ -514,29 +478,33 @@ async function fileExists(filePath: string): Promise { function resolveWatchDeps(deps: Partial = {}): AutoSyncWatchControlDeps { return { - isProcessAlive: - deps.isProcessAlive ?? - ((pid) => { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } - }), + isProcessAlive: deps.isProcessAlive ?? isProcessAlive, readProcessCommand: deps.readProcessCommand ?? ((pid) => { try { - const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'ignore'], - }).trim(); + const command = + process.platform === 'win32' + ? execFileSync( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `(Get-CimInstance Win32_Process -Filter \"ProcessId = ${pid}\").CommandLine`, + ], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim() + : execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); return command || undefined; } catch { return undefined; } }), + readProcessStartTime: deps.readProcessStartTime ?? readProcessStartTime, killProcess: deps.killProcess ?? ((pid, signal = 'SIGTERM') => { diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts index f38fce3f9..6fa1892b4 100644 --- a/gitnexus/src/core/auto-sync/state.ts +++ b/gitnexus/src/core/auto-sync/state.ts @@ -1,5 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; import { getGlobalDir } from '../../storage/repo-manager.js'; export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped'; @@ -19,6 +20,10 @@ export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string { return path.join(gitnexusDir, 'watch'); } +export function getAutoSyncMutexPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'watch.mutex'); +} + export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json'); } @@ -27,6 +32,26 @@ export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string { return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt'); } +export async function resetAutoSyncState(gitnexusDir = getGlobalDir()): Promise { + let releaseLock: () => Promise; + try { + releaseLock = await acquireFileLock(getAutoSyncMutexPath(gitnexusDir)); + } catch (error) { + if (error instanceof FileLockBusyError) return false; + throw error; + } + + try { + await Promise.all([ + fs.rm(getAutoSyncStatePath(gitnexusDir), { force: true }), + fs.rm(getProjectCommitInfoPath(gitnexusDir), { force: true }), + ]); + return true; + } finally { + await releaseLock(); + } +} + export function buildStateKey(repoPath: string, branch: string): string { return `${path.resolve(repoPath)}|${branch}`; } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 51af63b03..0fc2ac01b 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -52,6 +52,7 @@ import { shadowSidecarRecoveryMessage, } from './sidecar-recovery.js'; import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js'; +import { isProcessAlive } from '../../utils/process-identity.js'; import { logger } from '../logger.js'; // --------------------------------------------------------------------------- @@ -305,20 +306,6 @@ const INIT_LOCK_RETRY_DELAY_MS = 500; const initLockPath = (dbPath: string): string => `${dbPath}.init.lock`; -/** - * Returns true when the process identified by `pid` is still running. - * Uses `process.kill(pid, 0)` which sends signal 0 (a no-op probe) — - * it throws ESRCH when the process does not exist. - */ -const isProcessAlive = (pid: number): boolean => { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -}; - /** * Try to break a stale lock whose owning process has exited. * Returns `true` if the stale lock was removed (caller should retry acquire). diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index 5c35d69a3..4f3924057 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -20,6 +20,7 @@ import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; assertAnalysisFinalized: typeof import('../storage/repo-manager.js').assertAnalysisFinalized; + acquireAnalysisLock: (repoPath: string) => Promise<() => Promise>; send: (msg: WorkerMessage) => void; /** * Claim the single terminal-outcome slot. Returns `true` for the first caller @@ -44,33 +45,38 @@ export async function runWorkerAnalysis( ): Promise { let terminal: WorkerMessage; try { - const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap - ? [runnerIdentityAtBootstrap] - : []; - const result = await deps.runFullAnalysis( - repoPath, - // This worker force-exits right after reporting, so skip the native close - // (it can double-free in LadybugDB's ClientContext destructor after --pdg - // writes); flushWAL still persists the index, process.exit reclaims handles. - { ...options, skipNativeCloseOnExit: true }, - { - onProgress: (phase, percent, message) => - deps.send({ type: 'progress', phase, percent, message }), - onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), - }, - ...bootstrapArgs, - ); - // P2 (#2264): a half-finalized repo — meta.json written but the global - // registry entry missing (e.g. a prior collision-aborted run, or a wiped - // registry) — must NOT be reported as a successful analysis. Mirror the CLI's - // assertAnalysisFinalized guard so the worker surfaces it as an error instead - // of a false `complete` that leaves the repo invisible to list_repos. - await deps.assertAnalysisFinalized(repoPath); + const releaseAnalysisLock = await deps.acquireAnalysisLock(repoPath); + try { + const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap + ? [runnerIdentityAtBootstrap] + : []; + const result = await deps.runFullAnalysis( + repoPath, + // This worker force-exits right after reporting, so skip the native close + // (it can double-free in LadybugDB's ClientContext destructor after --pdg + // writes); flushWAL still persists the index, process.exit reclaims handles. + { ...options, skipNativeCloseOnExit: true }, + { + onProgress: (phase, percent, message) => + deps.send({ type: 'progress', phase, percent, message }), + onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), + }, + ...bootstrapArgs, + ); + // P2 (#2264): a half-finalized repo — meta.json written but the global + // registry entry missing (e.g. a prior collision-aborted run, or a wiped + // registry) — must NOT be reported as a successful analysis. Mirror the CLI's + // assertAnalysisFinalized guard so the worker surfaces it as an error instead + // of a false `complete` that leaves the repo invisible to list_repos. + await deps.assertAnalysisFinalized(repoPath); - // Send a JSON-safe projection, NOT the raw result: the IPC channel is - // default-JSON serialization and `result.pipelineResult` carries the live - // KnowledgeGraph. See analyze-worker-ipc.ts. - terminal = { type: 'complete', result: projectAnalyzeResultForIpc(result) }; + // Send a JSON-safe projection, NOT the raw result: the IPC channel is + // default-JSON serialization and `result.pipelineResult` carries the live + // KnowledgeGraph. See analyze-worker-ipc.ts. + terminal = { type: 'complete', result: projectAnalyzeResultForIpc(result) }; + } finally { + await releaseAnalysisLock(); + } } catch (err: unknown) { // Report the failure to the parent over IPC (the parent surfaces the message). const message = err instanceof Error ? err.message : 'Analysis failed'; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index 37ae1713b..1e0c89aae 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -11,6 +11,8 @@ * Child -> Parent: { type: 'error', message: string } */ +import path from 'path'; +import { createHash } from 'crypto'; import type { AnalyzeOptions } from '../core/run-analyze.js'; import { type AnalyzeResultIpc } from './analyze-worker-ipc.js'; import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js'; @@ -123,12 +125,13 @@ process.on('message', async (msg: StartMessage) => { const prepared = await identityModule.captureAnalyzerIdentityBeforeLoad( import.meta.url, async () => { - const [analysisModule, repoManager, shutdownHelpers] = await Promise.all([ + const [analysisModule, repoManager, shutdownHelpers, fileLock] = await Promise.all([ import('../core/run-analyze.js'), import('../storage/repo-manager.js'), import('../core/lbug/shutdown-helpers.js'), + import('../storage/file-lock.js'), ]); - return { analysisModule, repoManager, shutdownHelpers }; + return { analysisModule, repoManager, shutdownHelpers, fileLock }; }, ); boundedCheckpointBeforeExit = prepared.loaded.shutdownHelpers.boundedCheckpointBeforeExit; @@ -142,6 +145,18 @@ process.on('message', async (msg: StartMessage) => { { runFullAnalysis: prepared.loaded.analysisModule.runFullAnalysis, assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized, + acquireAnalysisLock: (repoPath) => { + const repoKey = createHash('sha256') + .update(prepared.loaded.repoManager.canonicalizePath(repoPath)) + .digest('hex'); + return prepared.loaded.fileLock.acquireFileLock( + path.join( + prepared.loaded.repoManager.getGlobalDir(), + 'locks', + `analyze-${repoKey}.lock`, + ), + ); + }, send, claimTerminal, }, diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 91df78f3c..c74b2d53f 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -9,6 +9,7 @@ import { spawn } from 'child_process'; import path from 'path'; import fs from 'fs/promises'; import { isIP } from 'net'; +import os from 'node:os'; import { logger } from '../core/logger.js'; import { getGlobalDir } from '../storage/repo-manager.js'; import { sanitizeRepoName } from '../storage/git.js'; @@ -416,27 +417,14 @@ export function normalizeGitUrlForCompare(url: string): string { * remote means for its threat model — for cloneOrPull, a missing remote * on an existing clone is treated as a refuse-to-pull condition. */ -export function getRemoteOriginUrl(cwd: string): Promise { - return new Promise((resolve) => { - const proc = spawn('git', ['config', '--get', 'remote.origin.url'], { - cwd, - stdio: ['ignore', 'pipe', 'pipe'], - windowsHide: true, - env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, - }); - let stdout = ''; - proc.stdout.on('data', (chunk: Buffer) => { - stdout += chunk; - }); - proc.on('close', (code) => { - if (code === 0 && stdout.trim()) { - resolve(stdout.trim()); - } else { - resolve(null); - } - }); - proc.on('error', () => resolve(null)); - }); +export async function getRemoteOriginUrl(cwd: string, timeoutMs?: number): Promise { + try { + const stdout = await runGit(['config', '--get', 'remote.origin.url'], cwd, { timeoutMs }); + return stdout.trim() || null; + } catch (error) { + if ((error as Error).message.includes('timed out')) throw error; + return null; + } } /** @@ -456,8 +444,9 @@ export function getRemoteOriginUrl(cwd: string): Promise { export async function assertRemoteMatchesRequestedUrl( targetDir: string, requestedUrl: string, + timeoutMs?: number, ): Promise { - const remoteUrl = await getRemoteOriginUrl(targetDir); + const remoteUrl = await getRemoteOriginUrl(targetDir, timeoutMs); if (remoteUrl === null) { throw new Error(`Existing clone at ${targetDir} has no remote.origin — refusing to pull`); } @@ -530,7 +519,7 @@ export async function cloneOrPull( if (options?.allowedCloneRoot) { await assertDirectoryOwnerAndPermissions(cloneRoot); } - await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); await assertPreRealpathContainment(cloneRoot, safeTarget); const exists = await fs.access(path.join(safeTarget, '.git')).then( @@ -544,11 +533,14 @@ export async function cloneOrPull( ); if (exists) { + if (options?.allowedCloneRoot) { + await assertNoSymlinkPath(cloneRoot, path.join(safeTarget, '.git'), true); + } await assertPostRealpathContainment(cloneRoot, safeTarget); // Confirm the existing clone is actually the same repository the caller // requested. Without this check, a pull would silently succeed against // whatever remote the dir was originally cloned from. - await assertRemoteMatchesRequestedUrl(safeTarget, url); + await assertRemoteMatchesRequestedUrl(safeTarget, url, options?.timeoutMs); onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); const runGitImpl = options?.runGitForTest ?? runGit; if (options?.branch) { @@ -602,11 +594,11 @@ export async function cloneOrPull( }); } } else { - if (targetExists) { + if (targetExists && (await fs.readdir(safeTarget)).length > 0) { throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); } await fs.mkdir(path.dirname(safeTarget), { recursive: true }); - await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); await assertPreRealpathContainment(cloneRoot, safeTarget); onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); try { @@ -622,12 +614,20 @@ export async function cloneOrPull( await assertPostRealpathContainment(cloneRoot, safeTarget); } catch (err: unknown) { if (options?.quarantineRoot) { - await fs - .access(safeTarget) - .then(async () => { - await quarantineAutoSyncPartial(safeTarget, options.quarantineRoot!); - }) - .catch(() => {}); + const partialExists = await fs.access(safeTarget).then( + () => true, + () => false, + ); + if (partialExists) { + try { + await quarantineAutoSyncPartial(safeTarget, options.quarantineRoot); + } catch (quarantineError) { + throw new AggregateError( + [err, quarantineError], + `Clone failed and partial checkout could not be quarantined: ${safeTarget}`, + ); + } + } } throw err; } @@ -654,7 +654,11 @@ async function assertPostRealpathContainment(root: string, target: string): Prom } } -async function assertNoSymlinkPath(root: string, target: string): Promise { +async function assertNoSymlinkPath( + root: string, + target: string, + verifyOwnership = false, +): Promise { const resolvedRoot = path.resolve(root); const resolvedTarget = path.resolve(target); const relativeTarget = path.relative(resolvedRoot, resolvedTarget); @@ -672,6 +676,7 @@ async function assertNoSymlinkPath(root: string, target: string): Promise if (stat.isSymbolicLink()) { throw new Error(`Refusing symlink in clone target path: ${current}`); } + if (verifyOwnership) await assertDirectoryOwnerAndPermissions(current); } } @@ -776,11 +781,10 @@ function warnIfCleartextCredential(url?: string): void { } /** - * Build the spawn env for `git`. Suppresses credential prompts and, when a - * credential resolves (see resolveGitCredential), injects a single - * host-scoped Authorization header via the `GIT_CONFIG_*` env protocol - * (git ≥2.31) so credentials never appear in argv or the URL. Appends after - * any existing `GIT_CONFIG_COUNT` rather than overwriting it. Exported for + * Build the spawn env for managed `git` commands. Suppresses credential + * prompts, disables repository hooks, and injects at most one host-scoped + * Authorization header via the `GIT_CONFIG_*` env protocol (git ≥2.31). + * Managed settings append after any existing GIT_CONFIG_COUNT. Exported for * unit tests. */ export function buildGitEnv( @@ -803,18 +807,21 @@ export function buildGitEnv( GIT_CURL_VERBOSE: undefined, }; + const existing = Number.parseInt(env.GIT_CONFIG_COUNT ?? '', 10); + let next = Number.isInteger(existing) && existing > 0 ? existing : 0; + env[`GIT_CONFIG_KEY_${next}`] = 'core.hooksPath'; + env[`GIT_CONFIG_VALUE_${next}`] = os.devNull; + next += 1; + const credential = resolveGitCredential(options); const key = options?.url ? buildExtraHeaderKey(options.url) : undefined; if (credential && key) { - // Append after any GIT_CONFIG_* the operator already set, so we never - // clobber their git config (e.g. an enforced http.sslVerify). - const existing = Number.parseInt(env.GIT_CONFIG_COUNT ?? '', 10); - const base = Number.isInteger(existing) && existing > 0 ? existing : 0; - env.GIT_CONFIG_COUNT = String(base + 1); - env[`GIT_CONFIG_KEY_${base}`] = key; - env[`GIT_CONFIG_VALUE_${base}`] = `Authorization: Basic ${credential}`; + env[`GIT_CONFIG_KEY_${next}`] = key; + env[`GIT_CONFIG_VALUE_${next}`] = `Authorization: Basic ${credential}`; + next += 1; warnIfCleartextCredential(options?.url); } + env.GIT_CONFIG_COUNT = String(next); return env; } @@ -824,7 +831,7 @@ export function buildGitEnv( // host-scoped Authorization header (GitHub PAT for github.com, else the // server's AZURE_DEVOPS_PAT for Azure hosts) via the GIT_CONFIG_* protocol — // never in argv. See resolveGitCredential / buildExtraHeaderKey. -function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { +export function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { return new Promise((resolve, reject) => { const spawnGit = options?.spawnForTest ?? spawn; const proc = spawnGit('git', args, { @@ -853,6 +860,9 @@ function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise< proc.kill('SIGTERM'); killTimer = setTimeout(() => { proc.kill('SIGKILL'); + finish(() => + reject(new Error(`git ${args[0]} timed out after ${options.timeoutMs}ms`)), + ); }, options.timeoutKillGraceMs ?? 1_000); }, options.timeoutMs) : undefined; diff --git a/gitnexus/src/storage/file-lock.ts b/gitnexus/src/storage/file-lock.ts new file mode 100644 index 000000000..2ed6a66cc --- /dev/null +++ b/gitnexus/src/storage/file-lock.ts @@ -0,0 +1,156 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { isProcessAlive, readProcessStartTime } from '../utils/process-identity.js'; + +export interface FileLockOptions { + retries?: number; + retryDelayMs?: number; + pid?: number; + processStartTime?: string; + isProcessAlive?: (pid: number) => boolean; + readProcessStartTime?: (pid: number) => string | undefined; +} + +interface FileLockOwner { + pid: number; + ownerId: string; + processStartTime: string; +} + +export class FileLockBusyError extends Error { + constructor(public readonly lockPath: string) { + super( + `Lock is already held: ${lockPath}. Confirm no owner process is active, then remove it manually.`, + ); + this.name = 'FileLockBusyError'; + } +} + +/** Acquire a recoverable cross-process mutex using an atomically published owner file. */ +export async function acquireFileLock( + lockPath: string, + options: FileLockOptions = {}, +): Promise<() => Promise> { + const resolvedPath = path.resolve(lockPath); + const retries = options.retries ?? 0; + const retryDelayMs = options.retryDelayMs ?? 50; + const pid = options.pid ?? process.pid; + const owner: FileLockOwner = { + pid, + ownerId: crypto.randomUUID(), + processStartTime: + options.processStartTime ?? (options.readProcessStartTime ?? readProcessStartTime)(pid) ?? '', + }; + if (!owner.processStartTime) { + throw new Error(`Unable to determine process start time for file lock owner pid ${owner.pid}.`); + } + + await fs.mkdir(path.dirname(resolvedPath), { recursive: true }); + const pendingPath = `${resolvedPath}.pending-${owner.ownerId}`; + await fs.writeFile(pendingPath, `${JSON.stringify(owner)}\n`, { encoding: 'utf-8', flag: 'wx' }); + + try { + for (let attempt = 0; ; attempt += 1) { + try { + await fs.link(pendingPath, resolvedPath); + break; + } catch (error) { + if (!isLockConflict(error)) throw error; + if ( + await reclaimStaleLock( + resolvedPath, + options.isProcessAlive ?? isProcessAlive, + options.readProcessStartTime ?? readProcessStartTime, + ) + ) { + continue; + } + if (attempt >= retries) throw new FileLockBusyError(lockPath); + await sleep(retryDelayMs); + } + } + } finally { + await fs.rm(pendingPath, { force: true }); + } + + let releasePromise: Promise | undefined; + return () => (releasePromise ??= releaseOwnedLock(resolvedPath, owner.ownerId)); +} + +async function reclaimStaleLock( + lockPath: string, + ownerIsAlive: (pid: number) => boolean, + getProcessStartTime: (pid: number) => string | undefined, +): Promise { + const reclaimGuardPath = `${lockPath}.reclaim`; + try { + await fs.mkdir(reclaimGuardPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') return false; + throw error; + } + + try { + const owner = await readOwner(lockPath); + if (!owner) return false; + if (ownerIsAlive(owner.pid)) { + const currentStartTime = getProcessStartTime(owner.pid); + if (!currentStartTime || currentStartTime === owner.processStartTime) return false; + } + + await fs.rm(lockPath, { force: true }); + return true; + } finally { + await fs.rmdir(reclaimGuardPath); + } +} + +async function releaseOwnedLock(lockPath: string, ownerId: string): Promise { + const releasePath = `${lockPath}.release-${ownerId}-${crypto.randomUUID()}`; + if (!(await moveOwnedLock(lockPath, releasePath, ownerId))) return; + await fs.rm(releasePath, { force: true }); +} + +async function moveOwnedLock( + lockPath: string, + destinationPath: string, + ownerId: string, +): Promise { + if ((await readOwner(lockPath))?.ownerId !== ownerId) return false; + try { + await fs.rename(lockPath, destinationPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } + + if ((await readOwner(destinationPath))?.ownerId === ownerId) return true; + await fs.rename(destinationPath, lockPath).catch(() => {}); + return false; +} + +async function readOwner(lockPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(lockPath, 'utf-8')) as Partial; + if ( + Number.isInteger(parsed.pid) && + Number(parsed.pid) > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime + ) { + return parsed as FileLockOwner; + } + } catch { + // Invalid or legacy locks fail closed; only verified dead owners are reclaimed. + } + return undefined; +} + +function isLockConflict(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + return code === 'EEXIST' || code === 'EPERM'; +} diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 568d288bd..987f8e46c 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -21,6 +21,7 @@ import os from 'os'; import { randomBytes } from 'crypto'; import { getInferredRepoName, resolveRepoIdentityRoot } from './git.js'; import { retryRename } from './fs-atomic.js'; +import { acquireFileLock } from './file-lock.js'; import { logger } from '../core/logger.js'; import { branchSlug, @@ -947,9 +948,25 @@ const writeRegistry = async (entries: RegistryEntry[]): Promise => { // truncated/half-written registry.json that the next load would treat as // empty and silently drop every registered repo (#2106 R9). const target = getGlobalRegistryPath(); - const tmp = `${target}.tmp`; - await fs.writeFile(tmp, JSON.stringify(entries, null, 2), 'utf-8'); - await fs.rename(tmp, target); + const tmp = `${target}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + try { + await fs.writeFile(tmp, JSON.stringify(entries, null, 2), 'utf-8'); + await fs.rename(tmp, target); + } finally { + await fs.unlink(tmp).catch(() => {}); + } +}; + +const withRegistryLock = async (operation: () => Promise): Promise => { + const release = await acquireFileLock(`${getGlobalRegistryPath()}.lock`, { + retries: 400, + retryDelayMs: 25, + }); + try { + return await operation(); + } finally { + await release(); + } }; /** @@ -1070,7 +1087,7 @@ const hasCustomAlias = (entry: RegistryEntry, inferredName: string | null): bool * caller can re-use it to keep AGENTS.md / skill files aligned with the * MCP-visible repo name (#979). */ -export const registerRepo = async ( +const registerRepoUnlocked = async ( repoPath: string, meta: RepoMeta, opts?: RegisterRepoOptions, @@ -1237,21 +1254,28 @@ export const registerRepo = async ( return name; }; +export const registerRepo = async ( + repoPath: string, + meta: RepoMeta, + opts?: RegisterRepoOptions, +): Promise => withRegistryLock(() => registerRepoUnlocked(repoPath, meta, opts)); + /** * Remove a repo from the global registry. * Called after `gitnexus clean`. */ -export const unregisterRepo = async (repoPath: string): Promise => { - // Canonicalise BOTH sides so an unregister call issued with the - // symlink form (`/var/folders/.../repo`) still matches an entry - // written with the realpath form (`/private/var/folders/.../repo`), - // and vice versa. Matches the semantics of `registerRepo` and - // `resolveRegistryEntry` post-#1003 review. - const resolved = canonicalizePath(repoPath); - const entries = await readRegistry(); - const filtered = entries.filter((e) => !registryPathEquals(canonicalizePath(e.path), resolved)); - await writeRegistry(filtered); -}; +export const unregisterRepo = async (repoPath: string): Promise => + withRegistryLock(async () => { + // Canonicalise BOTH sides so an unregister call issued with the + // symlink form (`/var/folders/.../repo`) still matches an entry + // written with the realpath form (`/private/var/folders/.../repo`), + // and vice versa. Matches the semantics of `registerRepo` and + // `resolveRegistryEntry` post-#1003 review. + const resolved = canonicalizePath(repoPath); + const entries = await readRegistry(); + const filtered = entries.filter((e) => !registryPathEquals(canonicalizePath(e.path), resolved)); + await writeRegistry(filtered); + }); /** * Remove a single non-primary branch's summary from a repo's registry entry @@ -1261,22 +1285,23 @@ export const unregisterRepo = async (repoPath: string): Promise => { * primary entry is left intact; an empty `branches[]` is dropped to keep the * registry shape legacy-clean. */ -export const removeBranchIndex = async (repoPath: string, branch: string): Promise => { - const resolved = canonicalizePath(repoPath); - const entries = await readRegistry(); - const idx = entries.findIndex((e) => registryPathEquals(canonicalizePath(e.path), resolved)); - if (idx < 0) return false; - const entry = entries[idx]; - const before = entry.branches?.length ?? 0; - if (!entry.branches || before === 0) return false; - const remaining = entry.branches.filter((b) => b.branch !== branch); - if (remaining.length === before) return false; // branch not recorded - if (remaining.length > 0) entry.branches = remaining; - else delete entry.branches; - entries[idx] = entry; - await writeRegistry(entries); - return true; -}; +export const removeBranchIndex = async (repoPath: string, branch: string): Promise => + withRegistryLock(async () => { + const resolved = canonicalizePath(repoPath); + const entries = await readRegistry(); + const idx = entries.findIndex((e) => registryPathEquals(canonicalizePath(e.path), resolved)); + if (idx < 0) return false; + const entry = entries[idx]; + const before = entry.branches?.length ?? 0; + if (!entry.branches || before === 0) return false; + const remaining = entry.branches.filter((b) => b.branch !== branch); + if (remaining.length === before) return false; // branch not recorded + if (remaining.length > 0) entry.branches = remaining; + else delete entry.branches; + entries[idx] = entry; + await writeRegistry(entries); + return true; + }); /** * Record that the flat workspace slot now serves `branch` (#2354). @@ -1348,18 +1373,20 @@ export const adoptFlatBranchLabel = async (repoPath: string, branch: string): Pr // multi-writer whole-file overwrite, and writing a pre-rm snapshot would // silently clobber concurrent registerRepo/removeBranchIndex writers — // the #2106 R9 re-read-before-write discipline registerRepo follows. - const entries = await readRegistry(); - const idx = isRegistered(entries); - if (idx < 0) return; // unregistered concurrently → still a no-op - const entry = entries[idx]; - const remaining = dirGone ? entry.branches?.filter((b) => b.branch !== branch) : entry.branches; - const droppedSummary = (entry.branches?.length ?? 0) !== (remaining?.length ?? 0); - if (entry.branch === branch && !droppedSummary) return; // already coherent - entry.branch = branch; - if (remaining && remaining.length > 0) entry.branches = remaining; - else delete entry.branches; - entries[idx] = entry; - await writeRegistry(entries); + await withRegistryLock(async () => { + const entries = await readRegistry(); + const idx = isRegistered(entries); + if (idx < 0) return; // unregistered concurrently → still a no-op + const entry = entries[idx]; + const remaining = dirGone ? entry.branches?.filter((b) => b.branch !== branch) : entry.branches; + const droppedSummary = (entry.branches?.length ?? 0) !== (remaining?.length ?? 0); + if (entry.branch === branch && !droppedSummary) return; // already coherent + entry.branch = branch; + if (remaining && remaining.length > 0) entry.branches = remaining; + else delete entry.branches; + entries[idx] = entry; + await writeRegistry(entries); + }); }; /** @@ -1651,6 +1678,7 @@ export const listRegisteredRepos = async (opts?: { // Validate each entry still has a .gitnexus/ directory with metadata const valid: RegistryEntry[] = []; + const prunedPaths: string[] = []; for (const entry of entries) { // Named to avoid shadowing the exported `hasIndex` function above. let indexFound = false; @@ -1681,6 +1709,7 @@ export const listRegisteredRepos = async (opts?: { valid.push(entry); } else if (!firstNonMissingError && lastMissingError) { // Index genuinely removed — safe to prune + prunedPaths.push(canonicalizePath(entry.path)); } else { // Not provably absent — keep entry to prevent mass registry wipe. // Warn so an I/O storm becomes observable instead of silently @@ -1693,9 +1722,17 @@ export const listRegisteredRepos = async (opts?: { } } - // If we pruned any entries, save the cleaned registry - if (valid.length !== entries.length) { - await writeRegistry(valid); + // Re-apply only the confirmed removals to a fresh snapshot while holding + // the registry lock; concurrent registrations must survive validation cleanup. + if (prunedPaths.length > 0) { + await withRegistryLock(async () => { + const fresh = await readRegistry(); + const cleaned = fresh.filter( + (entry) => + !prunedPaths.some((pruned) => registryPathEquals(canonicalizePath(entry.path), pruned)), + ); + if (cleaned.length !== fresh.length) await writeRegistry(cleaned); + }); } return valid; diff --git a/gitnexus/src/utils/process-identity.ts b/gitnexus/src/utils/process-identity.ts new file mode 100644 index 000000000..90972f0d4 --- /dev/null +++ b/gitnexus/src/utils/process-identity.ts @@ -0,0 +1,34 @@ +import { execFileSync } from 'node:child_process'; + +export function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } +} + +export function readProcessStartTime(pid: number): string | undefined { + try { + const startedAt = + process.platform === 'win32' + ? execFileSync( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; if ($p) { $p.CreationDate.ToUniversalTime().ToString("O") }`, + ], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim() + : execFileSync('ps', ['-p', String(pid), '-o', 'lstart='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return startedAt || undefined; + } catch { + return undefined; + } +} diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index f63ad08da..165ba3999 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -31,6 +31,7 @@ const baseResult: AnalyzeResult = { const okRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => baseResult); const okFinalize: WorkerAnalysisDeps['assertAnalysisFinalized'] = vi.fn(async () => undefined); +const okLock: WorkerAnalysisDeps['acquireAnalysisLock'] = vi.fn(async () => async () => undefined); const alwaysClaim: WorkerAnalysisDeps['claimTerminal'] = () => true; describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { @@ -48,6 +49,7 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized, + acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -69,6 +71,7 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized: okFinalize, + acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -89,6 +92,7 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: run, assertAnalysisFinalized: okFinalize, + acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -98,6 +102,31 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { expect(run.mock.calls[0]?.[3]).toBe(receipt); }); + it('does not enter analysis when another worker holds the repo lock', async () => { + const send = vi.fn<(msg: WorkerMessage) => void>(); + const run = vi.fn(async () => baseResult); + + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: run, + assertAnalysisFinalized: okFinalize, + acquireAnalysisLock: vi.fn(async () => { + throw new Error('Lock is already held for /repo'); + }), + send, + claimTerminal: alwaysClaim, + }, + ); + + expect(run).not.toHaveBeenCalled(); + expect(send).toHaveBeenCalledWith({ + type: 'error', + message: 'Lock is already held for /repo', + }); + }); + it('reports error when finalization passes but the analysis itself throws', async () => { const send = vi.fn<(msg: WorkerMessage) => void>(); const failingRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => { @@ -113,6 +142,7 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: failingRun, assertAnalysisFinalized: finalize, + acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -134,6 +164,7 @@ describe('runWorkerAnalysis — terminal-claim coordination (#2264 P3)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized: okFinalize, + acquireAnalysisLock: okLock, send, claimTerminal: alreadyClaimed, }, diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index f6e06f869..2908bfc32 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -3,6 +3,52 @@ import { describe, expect, it, vi } from 'vitest'; import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; describe('auto-sync analysis worker', () => { + it('ignores progress and resolves from the terminal complete message', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + stdout: { resume: vi.fn() }, + stderr: { resume: vi.fn() }, + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'progress', phase: 'parsing', progress: 20 }); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).resolves.toEqual({ stats: { files: 3 } }); + expect(child.stdout.resume).toHaveBeenCalled(); + expect(child.stderr.resume).toHaveBeenCalled(); + }); + + it('rejects immediately when the worker emits an error without exiting', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('error', new Error('spawn failed')); + + await expect(result).rejects.toThrow('Auto-sync analyze worker error: spawn failed'); + }); + + it('preserves a worker error after progress messages', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'progress', phase: 'parsing', progress: 20 }); + child.emit('message', { type: 'error', message: 'parser crashed' }); + child.emit('exit', 1, null); + + await expect(result).rejects.toThrow('parser crashed'); + }); it('waits for timed-out worker exit before releasing the scheduled run', async () => { const child = Object.assign(new EventEmitter(), { send: vi.fn(), diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 79501c3ed..8a924c877 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -43,6 +43,7 @@ const cloneRoot = { quarantineRetentionDays: 14, }; const verifiedWatchCommand = 'node /gitnexus/dist/cli/index.js watch'; +const verifiedProcessStartTime = 'Tue Aug 4 12:00:00 2026'; function withCloneRoot(deps: Partial): Partial { return { @@ -59,8 +60,12 @@ async function writeWatchOwner( await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); await fs.writeFile(paths.pidPath, `${pid}\n`); await fs.writeFile( - paths.lockPath, - `${JSON.stringify({ pid, ownerId, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + paths.mutexPath, + `${JSON.stringify({ pid, ownerId: `mutex-${ownerId}`, processStartTime: verifiedProcessStartTime })}\n`, + ); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid, ownerId, processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, ); await fs.writeFile( paths.statusPath, @@ -77,13 +82,13 @@ async function writeWatchOwner( describe('auto-sync runner', () => { it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-1', analyzedCommitId: 'commit-1', lastAnalyzeStatus: 'success', @@ -106,7 +111,7 @@ describe('auto-sync runner', () => { expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); expect(deps.cloneOrPull).toHaveBeenCalledWith( 'git@gitee.com:qts_server/qts_account.git', - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', undefined, { allowedCloneRoot: '/tmp/repos', @@ -118,14 +123,18 @@ describe('auto-sync runner', () => { overwriteLocalChanges: false, }, ); - expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); - expect(deps.runFullAnalysis).toHaveBeenCalledWith( - '/tmp/repos/qts_account', + expect(deps.getCurrentBranch).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + 10_000, + ); + expect(deps.runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', { branch: 'master', skipAgentsMd: true, skipSkills: true }, - { onProgress: expect.any(Function) }, + 1_800_000, + undefined, ); expect(deps.registerRepo).toHaveBeenCalledWith( - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', expect.objectContaining({ lastCommit: 'commit-2', branch: 'master' }), { name: 'gitee.com/qts_server/qts_account' }, ); @@ -142,10 +151,10 @@ describe('auto-sync runner', () => { it('syncs a group when a repo is newly added to the group', async () => { const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -170,13 +179,13 @@ describe('auto-sync runner', () => { it('syncs a group after successful re-analysis even when membership already exists', async () => { const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-3'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 2 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 2 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-2', analyzedCommitId: 'commit-2', lastAnalyzeStatus: 'success', @@ -230,7 +239,7 @@ describe('auto-sync runner', () => { cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), getCurrentBranch: vi.fn(() => 'main'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'service'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -247,13 +256,13 @@ describe('auto-sync runner', () => { expect(deps.registerRepo).toHaveBeenNthCalledWith( 1, - '/tmp/repos-a/service', + '/tmp/repos-a/github.com/team-a/service', expect.anything(), { name: 'github.com/team-a/service' }, ); expect(deps.registerRepo).toHaveBeenNthCalledWith( 2, - '/tmp/repos-b/service', + '/tmp/repos-b/gitlab.com/team-b/service', expect.anything(), { name: 'gitlab.com/team-b/service' }, ); @@ -271,13 +280,13 @@ describe('auto-sync runner', () => { it('skips analysis when commit id has not changed', async () => { const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-1'), - runFullAnalysis: vi.fn(), + runAnalysis: vi.fn(), registerRepo: vi.fn(), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-1', analyzedCommitId: 'commit-1', lastAnalyzeStatus: 'success', @@ -298,18 +307,24 @@ describe('auto-sync runner', () => { expect(result.analyzed).toBe(0); expect(result.skippedAnalysis).toBe(1); - expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.runAnalysis).not.toHaveBeenCalled(); expect(deps.syncGroupByName).not.toHaveBeenCalled(); }); - it('uses local_path plus repo name as the clone target', async () => { - expect(getConfiguredRepoPath(config.projects[0], 'qts_account')).toBe('/tmp/repos/qts_account'); + it('uses remote identity under local_path as the clone target', async () => { + expect( + getConfiguredRepoPath( + config.projects[0], + 'qts_account', + 'git@gitee.com:qts_server/qts_account.git', + ), + ).toBe('/tmp/repos/gitee.com/qts_server/qts_account'); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -326,7 +341,7 @@ describe('auto-sync runner', () => { expect(deps.cloneOrPull).toHaveBeenCalledWith( 'git@gitee.com:qts_server/qts_account.git', - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', undefined, { allowedCloneRoot: '/tmp/repos', @@ -344,7 +359,7 @@ describe('auto-sync runner', () => { const controller = new AbortController(); const runAnalysis = vi.fn(async () => ({ stats: { files: 1 } }) as any); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), runAnalysis, @@ -364,7 +379,7 @@ describe('auto-sync runner', () => { }); expect(runAnalysis).toHaveBeenCalledWith( - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', { branch: 'master', skipAgentsMd: true, skipSkills: true }, 1_800_000, controller.signal, @@ -381,11 +396,11 @@ describe('auto-sync runner', () => { const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async (_remoteUrl, _targetDir, _progress, options) => { if (options?.branch === 'missing') throw new Error('remote branch not found'); - return '/tmp/repos/qts_account'; + return '/tmp/repos/gitee.com/qts_server/qts_account'; }), getCurrentBranch: vi.fn(() => 'develop'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -404,21 +419,22 @@ describe('auto-sync runner', () => { expect(deps.cloneOrPull).toHaveBeenNthCalledWith( 1, 'git@gitee.com:qts_server/qts_account.git', - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', undefined, expect.objectContaining({ branch: 'missing' }), ); expect(deps.cloneOrPull).toHaveBeenNthCalledWith( 2, 'git@gitee.com:qts_server/qts_account.git', - '/tmp/repos/qts_account', + '/tmp/repos/gitee.com/qts_server/qts_account', undefined, expect.objectContaining({ branch: 'develop' }), ); - expect(deps.runFullAnalysis).toHaveBeenCalledWith( - '/tmp/repos/qts_account', + expect(deps.runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', { branch: 'develop', skipAgentsMd: true, skipSkills: true }, - { onProgress: expect.any(Function) }, + 1_800_000, + undefined, ); expect(warnLogger).toHaveBeenCalledWith( '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', @@ -439,7 +455,7 @@ describe('auto-sync runner', () => { }), getCurrentBranch: vi.fn(), getCurrentCommit: vi.fn(), - runFullAnalysis: vi.fn(), + runAnalysis: vi.fn(), registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -480,10 +496,10 @@ describe('auto-sync runner', () => { it('records branch_unavailable when checkout ends on an unexpected branch', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'develop'), getCurrentCommit: vi.fn(), - runFullAnalysis: vi.fn(), + runAnalysis: vi.fn(), registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -500,7 +516,7 @@ describe('auto-sync runner', () => { expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); - expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.runAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is develop; trying next branch.', @@ -510,10 +526,10 @@ describe('auto-sync runner', () => { it('records branch_unavailable when the checked out repository is detached', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => undefined), getCurrentCommit: vi.fn(), - runFullAnalysis: vi.fn(), + runAnalysis: vi.fn(), registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -530,7 +546,7 @@ describe('auto-sync runner', () => { expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); - expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.runAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is ; trying next branch.', @@ -554,11 +570,11 @@ describe('auto-sync runner', () => { const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async (remoteUrl) => { if (remoteUrl.includes('failing_sync')) throw new Error('sync failed'); - return '/tmp/repos/qts_account'; + return '/tmp/repos/gitee.com/qts_server/qts_account'; }), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => { + runAnalysis: vi.fn(async () => { throw new Error('analysis failed'); }), registerRepo: vi.fn(), @@ -587,7 +603,7 @@ describe('auto-sync runner', () => { expect(deps.syncGroupByName).not.toHaveBeenCalled(); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ - '/tmp/repos/qts_account|master': expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ codeCommitId: 'commit-2', lastAnalyzeStatus: 'failed', }), @@ -599,17 +615,94 @@ describe('auto-sync runner', () => { ), ); expect(errorLogger).toHaveBeenCalledWith( - expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), + expect.stringContaining('Analysis failed for /tmp/repos/gitee.com/qts_server/qts_account'), + ); + }); + + it('isolates clone-root resolution failures to the affected project', async () => { + const isolatedConfig: AutoSyncConfig = { + ...config, + projects: [ + { ...config.projects[0], localPath: '/bad/repos' }, + { ...config.projects[0], localPath: '/tmp/repos' }, + ], + }; + const deps: Partial = withCloneRoot({ + resolveCloneRoot: vi.fn(async (localPath: string) => { + if (localPath === '/bad/repos') throw new Error('unsafe clone root'); + return cloneRoot; + }), + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(isolatedConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).resolves.toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(1); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('persists state and commit info when repository registration fails', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => { + throw new Error('registry busy'); + }), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + lastAnalyzeStatus: 'failed', + analyzedCommitId: undefined, + lastAnalyzeError: 'Repository registration failed: registry busy', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Repository registration failed: registry busy', ); }); it('reports group sync failures after successful analysis', async () => { const errorLogger = vi.fn(); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -670,7 +763,7 @@ describe('auto-sync runner', () => { getCurrentCommit: vi.fn((repoPath) => repoPath.endsWith('/one') ? 'one-commit' : 'two-commit', ), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'repo'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), @@ -696,19 +789,24 @@ describe('auto-sync runner', () => { expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); }); - it('rejects duplicate resolved targetDir before clone work starts', async () => { + it('keeps same-basename remotes in distinct clone directories', async () => { const duplicateConfig: AutoSyncConfig = { ...config, maxConcurrency: 2, projects: [ { ...config.projects[0], + branches: ['main'], remoteUrls: ['git@github.com:owner/repo.git', 'git@gitlab.com:group/repo.git'], }, ], }; const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(), + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'main'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async (_path, _meta, options) => options?.name ?? 'repo'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), writeCommitInfo: vi.fn(async () => {}), @@ -722,11 +820,24 @@ describe('auto-sync runner', () => { deps, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, }), - ).rejects.toThrow('Duplicate auto-sync targetDir'); + ).resolves.toEqual({ synced: 2, analyzed: 2, skippedAnalysis: 0, failed: 0 }); - expect(deps.cloneOrPull).not.toHaveBeenCalled(); - expect(deps.saveState).not.toHaveBeenCalled(); - expect(deps.writeCommitInfo).not.toHaveBeenCalled(); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 1, + 'git@github.com:owner/repo.git', + '/tmp/repos/github.com/owner/repo', + undefined, + expect.any(Object), + ); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 2, + 'git@gitlab.com:group/repo.git', + '/tmp/repos/gitlab.com/group/repo', + undefined, + expect.any(Object), + ); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); }); it('rejects non auto-sync SSH URLs at runner boundary', async () => { @@ -756,15 +867,15 @@ describe('auto-sync runner', () => { it('increments analyze failure count and writes threshold details', async () => { const errorLogger = vi.fn(); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => { + runAnalysis: vi.fn(async () => { throw new Error('parser crashed\nwith stack'); }), registerRepo: vi.fn(), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-1', analyzedCommitId: 'commit-1', lastAnalyzeStatus: 'failed', @@ -789,7 +900,7 @@ describe('auto-sync runner', () => { expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ - '/tmp/repos/qts_account|master': expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ analyzeConsecutiveFailures: 1, lastAnalyzeError: 'parser crashed with stack', lastAnalyzeStatus: 'failed', @@ -805,20 +916,20 @@ describe('auto-sync runner', () => { }), ]); expect(errorLogger).toHaveBeenCalledWith( - '[auto-sync] Analysis failed for /tmp/repos/qts_account; consecutive failures 1/3: parser crashed with stack', + '[auto-sync] Analysis failed for /tmp/repos/gitee.com/qts_server/qts_account; consecutive failures 1/3: parser crashed with stack', ); }); it('retries analysis on a new commit after consecutive failures reached the threshold', async () => { const errorLogger = vi.fn(); const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-1', analyzedCommitId: 'commit-1', lastAnalyzeStatus: 'failed', @@ -841,10 +952,10 @@ describe('auto-sync runner', () => { }); expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); - expect(deps.runFullAnalysis).toHaveBeenCalledTimes(1); + expect(deps.runAnalysis).toHaveBeenCalledTimes(1); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ - '/tmp/repos/qts_account|master': expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ analyzeConsecutiveFailures: 0, lastAnalyzeError: undefined, lastAnalyzeStatus: 'success', @@ -864,13 +975,13 @@ describe('auto-sync runner', () => { it('clears prior analyze failure count after a successful analyze', async () => { const deps: Partial = withCloneRoot({ - cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), getCurrentBranch: vi.fn(() => 'master'), getCurrentCommit: vi.fn(() => 'commit-2'), - runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({ - '/tmp/repos/qts_account|master': { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { codeCommitId: 'commit-1', analyzedCommitId: 'commit-1', lastAnalyzeStatus: 'failed', @@ -895,7 +1006,7 @@ describe('auto-sync runner', () => { expect(result.analyzed).toBe(1); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ - '/tmp/repos/qts_account|master': expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ analyzeConsecutiveFailures: 0, lastAnalyzeError: undefined, lastAnalyzeStatus: 'success', @@ -1086,7 +1197,7 @@ describe('auto-sync starter', () => { expect(cancelled).toHaveBeenCalledTimes(1); await expect(fs.access(paths.pidPath)).rejects.toThrow(); - await expect(fs.access(paths.lockPath)).rejects.toThrow(); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; @@ -1106,6 +1217,7 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), }, }); @@ -1118,16 +1230,14 @@ describe('auto-sync starter', () => { } }); - it('removes stale pid and lock before starting watch', async () => { + it('recovers an abandoned watch mutex after the owner exits', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); - const timer = { unref: vi.fn() }; - const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; - const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + const stderr = { write: vi.fn() }; try { process.env.GITNEXUS_HOME = tempDir; - await writeWatchOwner(paths, 12345); + await writeWatchOwner(paths, 12345, 'abandoned-owner'); await fs.writeFile( path.join(tempDir, 'watch_config.yml'), [ @@ -1142,8 +1252,7 @@ describe('auto-sync starter', () => { const handle = await startAutoSyncWatch({ paths, - setIntervalFn, - clearIntervalFn, + stderr, runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), keepAlive: false, deps: { isProcessAlive: vi.fn(() => false) }, @@ -1151,7 +1260,9 @@ describe('auto-sync starter', () => { expect(handle).not.toBeNull(); expect(await fs.readFile(paths.pidPath, 'utf-8')).toBe(`${process.pid}\n`); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).not.toContain('abandoned-owner'); await handle?.stop(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; @@ -1159,7 +1270,7 @@ describe('auto-sync starter', () => { } }); - it('does not delete a half-initialized lock when pid has not been written yet', async () => { + it('does not delete a half-initialized lease when pid has not been written yet', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); @@ -1167,9 +1278,10 @@ describe('auto-sync starter', () => { try { process.env.GITNEXUS_HOME = tempDir; await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); await fs.writeFile( - paths.lockPath, - `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, ); await fs.writeFile( path.join(tempDir, 'watch_config.yml'), @@ -1190,6 +1302,7 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), }, }); @@ -1197,7 +1310,8 @@ describe('auto-sync starter', () => { expect(stderr.write).toHaveBeenCalledWith( '[auto-sync] Watch is already running with pid 12345.\n', ); - expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); await expect(fs.access(paths.pidPath)).rejects.toThrow(); } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; @@ -1206,16 +1320,17 @@ describe('auto-sync starter', () => { } }); - it('does not delete a live half-initialized lock when stop runs before pid is written', async () => { + it('does not delete a live half-initialized lease when stop runs before pid is written', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); const stderr = { write: vi.fn() }; const killProcess = vi.fn(); try { await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); await fs.writeFile( - paths.lockPath, - `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, ); await expect( @@ -1224,31 +1339,29 @@ describe('auto-sync starter', () => { stderr, deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, }), - ).resolves.toBe(false); + ).resolves.toBe('refused'); expect(killProcess).not.toHaveBeenCalled(); - expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); - await expect(fs.access(paths.pidPath)).rejects.toThrow(); - const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); - expect(status).toMatchObject({ - state: 'error', - pid: 12345, - ownerId: 'starting-owner', - message: expect.stringContaining('appears to be starting'), - }); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch appears to be starting with pid 12345; pid file is not ready.\n', + ); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.statusPath)).rejects.toThrow(); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); - it('removes a stale half-initialized lock when stop runs before pid is written', async () => { + it('does not delete a stale half-initialized lease from the stopper', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); try { await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); await fs.writeFile( - paths.lockPath, - `${JSON.stringify({ pid: 12345, ownerId: 'stale-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'stale-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, ); await expect( @@ -1261,22 +1374,31 @@ describe('auto-sync starter', () => { sleep: vi.fn(async () => {}), }, }), - ).resolves.toBe(false); + ).resolves.toBe('refused'); - await expect(fs.access(paths.lockPath)).rejects.toThrow(); - const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); - expect(status).toMatchObject({ - state: 'stale', - pid: 12345, - ownerId: 'stale-owner', - message: 'removed stale lock without pid file', - }); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('stale-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.statusPath)).rejects.toThrow(); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); - it('reports status and sends stop signals from pid files', async () => { + it('reports not_running when no watch lease exists', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + try { + await expect( + stopAutoSyncWatch({ + paths: getAutoSyncWatchPaths(tempDir), + stderr: { write: vi.fn() }, + }), + ).resolves.toBe('not_running'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports status and signals the verified owner without deleting its files', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); const killProcess = vi.fn(); @@ -1288,6 +1410,7 @@ describe('auto-sync starter', () => { readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), }), ).resolves.toMatchObject({ state: 'running', pid: 12345 }); await expect( @@ -1298,6 +1421,7 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => alive), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), killProcess: vi.fn((pid, signal) => { killProcess(pid, signal); alive = false; @@ -1305,18 +1429,101 @@ describe('auto-sync starter', () => { sleep: vi.fn(async () => {}), }, }), - ).resolves.toBe(true); + ).resolves.toBe('stopped'); expect(killProcess).toHaveBeenCalledWith(12345, 'SIGTERM'); - await expect(fs.access(paths.pidPath)).rejects.toThrow(); - await expect(fs.access(paths.lockPath)).rejects.toThrow(); - await expect(readAutoSyncWatchStatus(paths)).resolves.toMatchObject({ state: 'stopped' }); + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + expect(JSON.parse(await fs.readFile(paths.statusPath, 'utf-8'))).toMatchObject({ + state: 'running', + pid: 12345, + }); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); - it('does not mark stopped when stop times out waiting for the owner process', async () => { + it('does not delete or overwrite successor ownership after the old owner exits', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + let oldOwnerAlive = true; + let handedOver = false; + try { + await writeWatchOwner(paths, 12345, 'old-owner'); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + pollMs: 1, + deps: { + isProcessAlive: vi.fn((pid) => (pid === 12345 ? oldOwnerAlive : true)), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + killProcess: vi.fn(), + sleep: vi.fn(async () => { + if (handedOver) return; + handedOver = true; + oldOwnerAlive = false; + await fs.writeFile(paths.pidPath, '54321\n'); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid: 54321, ownerId: 'successor', processStartTime: verifiedProcessStartTime, createdAt: '2026-08-04T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ state: 'running', pid: 54321, ownerId: 'successor', updatedAt: '2026-08-04T00:00:00.000Z' })}\n`, + ); + }), + }, + }), + ).resolves.toBe('stopped'); + + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('54321\n'); + expect(JSON.parse(await fs.readFile(paths.ownerPath, 'utf-8'))).toMatchObject({ + pid: 54321, + ownerId: 'successor', + }); + expect(JSON.parse(await fs.readFile(paths.statusPath, 'utf-8'))).toMatchObject({ + state: 'running', + pid: 54321, + ownerId: 'successor', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses to signal when the process command is unavailable', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + pollMs: 1, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => undefined), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + killProcess, + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe('refused'); + expect(killProcess).not.toHaveBeenCalled(); + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not change owner status when stop times out', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); try { @@ -1331,23 +1538,23 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), killProcess: vi.fn(), sleep: vi.fn(async () => {}), }, }), - ).resolves.toBe(false); + ).resolves.toBe('timeout'); await expect( readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), }), - ).resolves.toMatchObject({ - state: 'stopping', - pid: 12345, - message: expect.stringContaining('did not exit'), - }); + ).resolves.toMatchObject({ state: 'running', pid: 12345 }); await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -1361,8 +1568,8 @@ describe('auto-sync starter', () => { await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); await fs.writeFile(paths.pidPath, '12345\n'); await fs.writeFile( - paths.lockPath, - `${JSON.stringify({ pid: 12345, ownerId: 'lock-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'lock-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, ); await fs.writeFile( paths.statusPath, @@ -1375,7 +1582,7 @@ describe('auto-sync starter', () => { stderr: { write: vi.fn() }, deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, }), - ).resolves.toBe(false); + ).resolves.toBe('refused'); expect(killProcess).not.toHaveBeenCalled(); await expect( @@ -1390,6 +1597,44 @@ describe('auto-sync starter', () => { } }); + it('refuses to signal a reused pid even when it is another GitNexus watch', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => 'Tue Aug 4 13:00:00 2026'), + killProcess, + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe('refused'); + + expect(killProcess).not.toHaveBeenCalled(); + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => 'Tue Aug 4 13:00:00 2026'), + }), + ).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('different process'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it('refuses to signal a reused pid whose command is not GitNexus watch', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); @@ -1404,17 +1649,19 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), killProcess, sleep: vi.fn(async () => {}), }, }), - ).resolves.toBe(false); + ).resolves.toBe('refused'); expect(killProcess).not.toHaveBeenCalled(); await expect( readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), }), ).resolves.toMatchObject({ state: 'error', @@ -1426,17 +1673,23 @@ describe('auto-sync starter', () => { } }); - it('restart can start only after stop confirms pid and lock cleanup', async () => { + it('restart starts only after the owner releases its mutex', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); const timer = { unref: vi.fn() }; const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + const runOnce = vi.fn(async () => ({ + synced: 0, + analyzed: 0, + skippedAnalysis: 0, + failed: 0, + })); let alive = true; + let ownerStop: Promise | undefined; try { process.env.GITNEXUS_HOME = tempDir; - await writeWatchOwner(paths, 12345); await fs.writeFile( path.join(tempDir, 'watch_config.yml'), [ @@ -1448,6 +1701,15 @@ describe('auto-sync starter', () => { ' - git@github.com:team/repo.git', ].join('\n'), ); + const ownerHandle = await startAutoSyncWatch({ + paths, + setIntervalFn, + clearIntervalFn, + runOnce, + keepAlive: false, + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + expect(ownerHandle).not.toBeNull(); await expect( stopAutoSyncWatch({ @@ -1458,26 +1720,31 @@ describe('auto-sync starter', () => { deps: { isProcessAlive: vi.fn(() => alive), readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), killProcess: vi.fn(() => { - alive = false; + ownerStop = ownerHandle!.stop().then(() => { + alive = false; + }); + }), + sleep: vi.fn(async () => { + await ownerStop; }), - sleep: vi.fn(async () => {}), }, }), - ).resolves.toBe(true); + ).resolves.toBe('stopped'); await expect(fs.access(paths.pidPath)).rejects.toThrow(); - await expect(fs.access(paths.lockPath)).rejects.toThrow(); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); - const handle = await startAutoSyncWatch({ + const successorHandle = await startAutoSyncWatch({ paths, setIntervalFn, clearIntervalFn, - runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + runOnce, keepAlive: false, - deps: { isProcessAlive: vi.fn(() => false) }, }); - expect(handle).not.toBeNull(); - await handle?.stop(); + expect(successorHandle).not.toBeNull(); + await successorHandle?.stop(); } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index cca2365e7..940f23871 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -5,20 +5,25 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { extractRepoNameFromRemoteUrl, + getAutoSyncMutexPath, getAutoSyncStatePath, getAutoSyncWatchDir, getProjectCommitInfoPath, loadAutoSyncConfig, + parseAutoSyncConfig, parseBranchCandidates, parseDurationMs, + quarantineAutoSyncPartial, resolveConfiguredCloneRoot, loadAutoSyncState, + resetAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, validateAutoSyncRemoteUrl, validateAutoSyncBranchName, writeProjectCommitInfo, } from '../../src/core/auto-sync/index.js'; +import { acquireFileLock } from '../../src/storage/file-lock.js'; describe('auto-sync', () => { let tempDir: string; @@ -44,6 +49,9 @@ describe('auto-sync', () => { it('places watch runtime artifacts under the watch directory by default', () => { expect(getAutoSyncWatchDir(gitnexusHome)).toBe(path.join(gitnexusHome, 'watch')); + expect(getAutoSyncMutexPath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'watch.mutex'), + ); expect(getAutoSyncStatePath(gitnexusHome)).toBe( path.join(gitnexusHome, 'watch', 'auto-sync-state.json'), ); @@ -52,6 +60,38 @@ describe('auto-sync', () => { ); }); + it('refuses to reset state while the watch mutex is held', async () => { + const statePath = getAutoSyncStatePath(gitnexusHome); + const infoPath = getProjectCommitInfoPath(gitnexusHome); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, '{"kept":true}\n'); + await fs.writeFile(infoPath, 'kept\n'); + const release = await acquireFileLock(getAutoSyncMutexPath(gitnexusHome)); + + try { + await expect(resetAutoSyncState(gitnexusHome)).resolves.toBe(false); + await expect(fs.readFile(statePath, 'utf-8')).resolves.toContain('kept'); + await expect(fs.readFile(infoPath, 'utf-8')).resolves.toBe('kept\n'); + } finally { + await release(); + } + }); + + it('resets derived state while holding the watch mutex', async () => { + const statePath = getAutoSyncStatePath(gitnexusHome); + const infoPath = getProjectCommitInfoPath(gitnexusHome); + const mutexPath = getAutoSyncMutexPath(gitnexusHome); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, '{}\n'); + await fs.writeFile(infoPath, 'derived\n'); + + await expect(resetAutoSyncState(gitnexusHome)).resolves.toBe(true); + + await expect(fs.access(statePath)).rejects.toThrow(); + await expect(fs.access(infoPath)).rejects.toThrow(); + await expect(fs.access(mutexPath)).rejects.toThrow(); + }); + it('loads watch_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { await fs.writeFile( path.join(gitnexusHome, 'watch_config.yml'), @@ -116,6 +156,23 @@ describe('auto-sync', () => { expect(loaded.config.projects[0].overwriteLocalChanges).toBe(false); }); + it('rejects repo_git_timeout values above the Node timer limit', () => { + expect(() => + parseAutoSyncConfig( + [ + 'sync_interval_minutes: 10', + 'repo_git_timeout: 2147483648ms', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: main', + ' remote_urls:', + ' - https://github.com/owner/repo.git', + ].join('\n'), + '/tmp/watch_config.yml', + ), + ).toThrow('repo_git_timeout must not exceed 2147483647ms'); + }); + it('rejects analyze_timeout values above half the sync interval', async () => { await fs.writeFile( path.join(gitnexusHome, 'watch_config.yml'), @@ -258,6 +315,60 @@ describe('auto-sync', () => { ); }); + it('removes expired quarantine entries while preserving recent and unrelated files', async () => { + const root = path.join(tempDir, 'repos'); + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + const expired = path.join(quarantineRoot, 'auto-sync-expired-repo'); + const recent = path.join(quarantineRoot, 'auto-sync-recent-repo'); + const unrelated = path.join(quarantineRoot, 'operator-note.txt'); + await fs.mkdir(expired, { recursive: true }); + await fs.mkdir(recent); + await fs.writeFile(unrelated, 'keep'); + const old = new Date(Date.now() - 15 * 24 * 60 * 60 * 1_000); + await fs.utimes(expired, old, old); + + await resolveConfiguredCloneRoot(root); + + await expect(fs.access(expired)).rejects.toThrow(); + await expect(fs.access(recent)).resolves.toBeUndefined(); + await expect(fs.readFile(unrelated, 'utf-8')).resolves.toBe('keep'); + }); + + it('falls back to copy and remove when quarantine crosses filesystems', async () => { + const target = path.join(tempDir, 'partial-repo'); + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'partial.txt'), 'partial'); + vi.spyOn(fs, 'rename').mockRejectedValueOnce( + Object.assign(new Error('cross-device link'), { code: 'EXDEV' }), + ); + + const destination = await quarantineAutoSyncPartial(target, quarantineRoot); + + await expect(fs.readFile(path.join(destination, 'partial.txt'), 'utf-8')).resolves.toBe( + 'partial', + ); + await expect(fs.access(target)).rejects.toThrow(); + }); + + it('rejects group-writable configured clone roots', async () => { + if (process.platform === 'win32') return; + const root = path.join(tempDir, 'group-writable-repos'); + await fs.mkdir(root, { mode: 0o770 }); + await fs.chmod(root, 0o770); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('group-writable'); + }); + + it('rejects sticky world-writable configured clone roots', async () => { + if (process.platform === 'win32') return; + const root = path.join(tempDir, 'sticky-world-writable-repos'); + await fs.mkdir(root); + await fs.chmod(root, 0o1777); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('world-writable'); + }); + it('creates missing configured clone roots before watch clone work', async () => { const root = path.join(tempDir, 'missing-repos'); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 97ef63567..500db326e 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -1,5 +1,6 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { Command, Option } from 'commander'; @@ -251,14 +252,14 @@ describe('CLI help surface', () => { expect(result.status).toBe(0); expect(result.stdout).toContain('gitnexus watch [options] [action]'); - expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status'); + expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status, reset'); expect(result.stdout).toContain('GITNEXUS_HOME/watch_config.yml'); expect(result.stdout).toContain('GITNEXUS_HOME/watch/watch.pid'); expect(result.stdout).toContain('GITNEXUS_HOME/watch/project_commit_info.txt'); }); it('watch init creates the default watch_config.yml and does not overwrite it', () => { - const home = fs.mkdtempSync(path.join(repoRoot, '.tmp-test/gitnexus-watch-init-')); + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-init-')); try { const first = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); const configPath = path.join(home, 'watch_config.yml'); @@ -270,9 +271,10 @@ describe('CLI help surface', () => { expect(config).toContain('analyze_failure_threshold: 3'); expect(config).toContain('analyze_timeout: 5m'); expect(config).toContain('overwrite_local_changes: false'); - expect(config).toContain(`local_path: ${path.join(home, 'repo')}`); + expect(config).toContain(`local_path: ${path.join(home, 'repos')}`); expect(config).not.toContain('/abs/path/to/repos'); expect(config).toContain('git@github.com:owner/repo.git'); + expect(config).not.toContain('group_name:'); const second = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); @@ -284,6 +286,52 @@ describe('CLI help surface', () => { } }); + it('watch reset removes only derived auto-sync state files', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-reset-')); + const watchDir = path.join(home, 'watch'); + const cloneMarker = path.join(home, 'repos', 'repo', 'keep.txt'); + try { + fs.mkdirSync(path.dirname(cloneMarker), { recursive: true }); + fs.writeFileSync(cloneMarker, 'keep'); + fs.mkdirSync(watchDir, { recursive: true }); + fs.writeFileSync(path.join(watchDir, 'auto-sync-state.json'), '{}'); + fs.writeFileSync(path.join(watchDir, 'project_commit_info.txt'), 'derived'); + + const result = runCliArgs(['watch', 'reset'], { GITNEXUS_HOME: home }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('Reset analysis state'); + expect(fs.existsSync(path.join(watchDir, 'auto-sync-state.json'))).toBe(false); + expect(fs.existsSync(path.join(watchDir, 'project_commit_info.txt'))).toBe(false); + expect(fs.readFileSync(cloneMarker, 'utf8')).toBe('keep'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it('watch stop exits non-zero when no watch was stopped', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-stop-')); + try { + const result = runCliArgs(['watch', 'stop'], { GITNEXUS_HOME: home }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('Watch is not running'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it('watch restart starts when the watch is not running', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-restart-')); + try { + const result = runCliArgs(['watch', 'restart'], { GITNEXUS_HOME: home }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('Watch is not running'); + expect(result.stderr).toContain('Missing config file'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it('wiki help shows provider, review, and verbose flags', () => { const result = runHelp('wiki'); diff --git a/gitnexus/test/unit/file-lock.test.ts b/gitnexus/test/unit/file-lock.test.ts new file mode 100644 index 000000000..b02704b10 --- /dev/null +++ b/gitnexus/test/unit/file-lock.test.ts @@ -0,0 +1,154 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { acquireFileLock, FileLockBusyError } from '../../src/storage/file-lock.js'; + +const tempDirs: string[] = []; + +async function tempLockPath(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-file-lock-')); + tempDirs.push(dir); + return path.join(dir, 'locks', 'test.mutex'); +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +describe('file lock', () => { + it('rejects a second holder for the same path', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + + await expect(acquireFileLock(lockPath)).rejects.toBeInstanceOf(FileLockBusyError); + + await release(); + }); + + it('releases idempotently', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + + await release(); + await expect(release()).resolves.toBeUndefined(); + const nextRelease = await acquireFileLock(lockPath); + await nextRelease(); + }); + + it('reclaims a lock whose owner process exited', async () => { + const lockPath = await tempLockPath(); + const oldRelease = await acquireFileLock(lockPath, { + pid: 111, + processStartTime: 'old-start', + }); + + const nextRelease = await acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'new-start', + isProcessAlive: () => false, + }); + + await oldRelease(); + await expect( + acquireFileLock(lockPath, { + isProcessAlive: (pid) => pid === 222, + readProcessStartTime: () => 'new-start', + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await nextRelease(); + }); + + it('reclaims a reused pid only when its start time differs', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 111, processStartTime: 'old-start' }); + + const nextRelease = await acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'next-start', + isProcessAlive: () => true, + readProcessStartTime: () => 'reused-pid-start', + }); + + await nextRelease(); + }); + + it('fails closed for a legacy or invalid lock without owner metadata', async () => { + const lockPath = await tempLockPath(); + await fs.mkdir(lockPath, { recursive: true }); + + await expect(acquireFileLock(lockPath)).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.access(lockPath)).resolves.toBeUndefined(); + }); + + it('waits for the current holder when retries are configured', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + const next = acquireFileLock(lockPath, { retries: 20, retryDelayMs: 5 }); + + await sleep(10); + await release(); + const nextRelease = await next; + + await nextRelease(); + }); + + it('fails closed while another stale-lock recovery is in progress', async () => { + const lockPath = await tempLockPath(); + const oldRelease = await acquireFileLock(lockPath, { + pid: 999, + processStartTime: 'abandoned', + }); + const reclaimGuardPath = `${lockPath}.reclaim`; + await fs.mkdir(reclaimGuardPath); + + await expect( + acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.access(lockPath)).resolves.toBeUndefined(); + + await fs.rmdir(reclaimGuardPath); + const nextRelease = await acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }); + await oldRelease(); + await nextRelease(); + }); + + it('allows only one contender to recover an abandoned lock', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 999, processStartTime: 'abandoned' }); + const starts = new Map( + Array.from({ length: 8 }, (_, index) => [1000 + index, `start-${index}`]), + ); + + const results = await Promise.allSettled( + [...starts].map(([pid, processStartTime]) => + acquireFileLock(lockPath, { + pid, + processStartTime, + isProcessAlive: (ownerPid) => ownerPid !== 999, + readProcessStartTime: (ownerPid) => starts.get(ownerPid), + }), + ), + ); + + const acquired = results.filter( + (result): result is PromiseFulfilledResult<() => Promise> => + result.status === 'fulfilled', + ); + expect(acquired).toHaveLength(1); + for (const result of results) { + if (result.status === 'rejected') expect(result.reason).toBeInstanceOf(FileLockBusyError); + } + await acquired[0].value(); + }); +}); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 1b57b8347..ac3d3b396 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -404,7 +404,7 @@ describe('git-clone', () => { }); }); - describe('buildGitEnv — token injection', () => { + describe('buildGitEnv — managed git environment', () => { // The token MUST travel via GIT_CONFIG_* env vars (git ≥2.31), not via // argv or URL. This keeps it out of `ps`, shell history, and stderr. @@ -428,33 +428,33 @@ describe('git-clone', () => { expect(env.GIT_CURL_VERBOSE).toBeUndefined(); }); - it('does not set GIT_CONFIG_* env vars when no token is provided', () => { + it('disables repository hooks even when no token is provided', () => { const env = buildGitEnv({}); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); - expect(env.GIT_CONFIG_KEY_0).toBeUndefined(); - expect(env.GIT_CONFIG_VALUE_0).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_KEY_0).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_0).toBe(os.devNull); }); - it('also leaves GIT_CONFIG_* unset when token is empty string', () => { + it('only disables repository hooks when token is empty string', () => { const env = buildGitEnv({}, { token: '' }); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); - expect(env.GIT_CONFIG_KEY_0).toBeUndefined(); - expect(env.GIT_CONFIG_VALUE_0).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_KEY_0).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_0).toBe(os.devNull); }); it('injects a host-scoped Basic-auth header when a github.com token is provided', () => { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://github.com/owner/repo' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_COUNT).toBe('2'); // Host-scoped key: the header attaches only to this origin's requests. - expect(env.GIT_CONFIG_KEY_0).toBe('http.https://github.com/owner/repo.extraHeader'); + expect(env.GIT_CONFIG_KEY_1).toBe('http.https://github.com/owner/repo.extraHeader'); const expected = 'Authorization: Basic ' + Buffer.from('x-access-token:ghp_secret123').toString('base64'); - expect(env.GIT_CONFIG_VALUE_0).toBe(expected); + expect(env.GIT_CONFIG_VALUE_1).toBe(expected); }); it('does not inject a token for a non-github host (defense-in-depth host bind)', () => { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://gitlab.com/owner/repo' }); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); }); it('never includes the raw token value in any env entry', () => { @@ -463,7 +463,7 @@ describe('git-clone', () => { const token = 'ghp_uniqueRawSecret_98765'; const env = buildGitEnv({ EXISTING: 'value' }, { token, url: 'https://github.com/o/r' }); for (const [key, value] of Object.entries(env)) { - if (key === 'GIT_CONFIG_VALUE_0') continue; + if (key === 'GIT_CONFIG_VALUE_1') continue; expect(String(value)).not.toContain(token); } }); @@ -473,12 +473,12 @@ describe('git-clone', () => { process.env.AZURE_DEVOPS_PAT = 'azure-pat-xyz'; try { const env = buildGitEnv({}, { url: 'https://dev.azure.com/org/proj/_git/repo' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); - expect(env.GIT_CONFIG_KEY_0).toBe( + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_KEY_1).toBe( 'http.https://dev.azure.com/org/proj/_git/repo.extraHeader', ); const expected = 'Authorization: Basic ' + Buffer.from(':azure-pat-xyz').toString('base64'); - expect(env.GIT_CONFIG_VALUE_0).toBe(expected); + expect(env.GIT_CONFIG_VALUE_1).toBe(expected); } finally { if (prev === undefined) delete process.env.AZURE_DEVOPS_PAT; else process.env.AZURE_DEVOPS_PAT = prev; @@ -492,8 +492,8 @@ describe('git-clone', () => { process.env.AZURE_DEVOPS_PAT = 'azure-pat-xyz'; try { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://github.com/o/r' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); - expect(env.GIT_CONFIG_VALUE_1).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_VALUE_2).toBeUndefined(); for (const value of Object.values(env)) { expect(String(value)).not.toContain('azure-pat-xyz'); } @@ -508,13 +508,48 @@ describe('git-clone', () => { { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.sslVerify', GIT_CONFIG_VALUE_0: 'true' }, { token: 'ghp_secret123', url: 'https://github.com/o/r' }, ); - expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_COUNT).toBe('3'); // Operator's pre-existing config is preserved at index 0. expect(env.GIT_CONFIG_KEY_0).toBe('http.sslVerify'); expect(env.GIT_CONFIG_VALUE_0).toBe('true'); - // Our credential is appended at index 1. - expect(env.GIT_CONFIG_KEY_1).toBe('http.https://github.com/o/r.extraHeader'); - expect(env.GIT_CONFIG_VALUE_1).toContain('Authorization: Basic '); + expect(env.GIT_CONFIG_KEY_1).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_1).toBe(os.devNull); + expect(env.GIT_CONFIG_KEY_2).toBe('http.https://github.com/o/r.extraHeader'); + expect(env.GIT_CONFIG_VALUE_2).toContain('Authorization: Basic '); + }); + + it('overrides an inherited hooks path with the managed safe value', () => { + const env = buildGitEnv({ + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.hooksPath', + GIT_CONFIG_VALUE_0: '/tmp/untrusted-hooks', + }); + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_KEY_1).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_1).toBe(os.devNull); + }); + + it('does not execute hooks from an existing repository', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-managed-git-'); + const marker = path.join(root, 'hook-ran'); + try { + await runGit(['init', '--initial-branch=main'], root); + await runGit(['config', 'user.email', 'test@example.com'], root); + await runGit(['config', 'user.name', 'GitNexus Test'], root); + await fs.writeFile(path.join(root, 'README.md'), 'test\n'); + await runGit(['add', 'README.md'], root); + await runGit(['commit', '-m', 'initial'], root); + const hook = path.join(root, '.git', 'hooks', 'post-checkout'); + await fs.writeFile(hook, `#!/bin/sh\ntouch ${JSON.stringify(marker)}\n`); + await fs.chmod(hook, 0o700); + + await runGitForTest(['checkout', '-b', 'next'], root); + + await expect(fs.access(marker)).rejects.toThrow(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } }); it('strips control characters from the config key (no key injection)', () => { @@ -522,7 +557,7 @@ describe('git-clone', () => { {}, { token: 'ghp_secret123', url: 'https://github.com/o/r%0Anewline' }, ); - const key = env.GIT_CONFIG_KEY_0 ?? ''; + const key = env.GIT_CONFIG_KEY_1 ?? ''; expect(key).not.toContain('\n'); expect(key).not.toContain('\r'); }); @@ -686,6 +721,69 @@ describe('git-clone', () => { } }); + it('rejects writable existing directories below a controlled clone root', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const namespace = path.join(root, 'team'); + try { + await fs.mkdir(namespace); + await fs.chmod(namespace, 0o777); + await expect( + cloneOrPull( + 'https://example.com/team/repo.git', + path.join(namespace, 'repo'), + undefined, + { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }, + ), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(namespace, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects writable .git metadata in an existing controlled clone', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const gitDir = path.join(target, '.git'); + try { + await fs.mkdir(gitDir, { recursive: true }); + await fs.chmod(gitDir, 0o777); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(gitDir, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked .git metadata in an existing controlled clone', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const outside = await mkControlledRoot('gitnexus-outside-git-dir-'); + const target = path.join(root, 'repo'); + try { + await fs.mkdir(target); + await fs.symlink(outside, path.join(target, '.git')); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('symlink'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); + } + }); + it('rejects existing clones whose remote origin mismatches the requested URL', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); const target = path.join(root, 'repo'); @@ -796,6 +894,25 @@ describe('git-clone', () => { } }); + it('clones into a pre-existing empty target directory', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const runGitForTest = vi.fn(async () => ''); + try { + await fs.mkdir(target); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + runGitForTest, + }), + ).resolves.toBe(target); + expect(runGitForTest).toHaveBeenCalled(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('quarantines partial auto-sync clone output on clone failure', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); const quarantineRoot = path.join(root, 'quarantine'); @@ -1179,7 +1296,7 @@ describe('git-clone', () => { }); describe('runGit timeout', () => { - it('waits for close and sends SIGKILL after the grace period before returning timeout', async () => { + it('rejects after SIGKILL even when the child never closes', async () => { vi.useFakeTimers(); try { const child = new EventEmitter() as EventEmitter & { @@ -1209,7 +1326,6 @@ describe('git-clone', () => { await vi.advanceTimersByTimeAsync(25); expect(child.kill).toHaveBeenCalledWith('SIGKILL'); - child.emit('close', null); await expect(promise).rejects.toThrow('timed out after 20ms'); } finally { vi.useRealTimers(); diff --git a/gitnexus/test/unit/process-identity.test.ts b/gitnexus/test/unit/process-identity.test.ts new file mode 100644 index 000000000..9e337fd18 --- /dev/null +++ b/gitnexus/test/unit/process-identity.test.ts @@ -0,0 +1,22 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { isProcessAlive } from '../../src/utils/process-identity.js'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('process identity', () => { + it('treats only ESRCH as a dead process', () => { + const kill = vi.spyOn(process, 'kill'); + kill.mockImplementationOnce(() => { + throw Object.assign(new Error('missing'), { code: 'ESRCH' }); + }); + kill.mockImplementationOnce(() => { + throw Object.assign(new Error('not permitted'), { code: 'EPERM' }); + }); + + expect(isProcessAlive(111)).toBe(false); + expect(isProcessAlive(222)).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/repo-manager.test.ts b/gitnexus/test/unit/repo-manager.test.ts index a269daab2..1ae587aea 100644 --- a/gitnexus/test/unit/repo-manager.test.ts +++ b/gitnexus/test/unit/repo-manager.test.ts @@ -771,6 +771,25 @@ describe('registerRepo name override + collision guard (#829)', () => { expect(entries[0].name).not.toBe(path.basename(tmpRepoA.dbPath)); }); + it('preserves every concurrent registration', async () => { + const repoPaths = Array.from({ length: 12 }, (_, index) => + path.join(tmpRepoA.dbPath, `concurrent-${index}`), + ); + await Promise.all(repoPaths.map((repoPath) => fs.mkdir(repoPath, { recursive: true }))); + + await Promise.all( + repoPaths.map((repoPath, index) => + registerRepo(repoPath, meta, { name: `concurrent-${index}` }), + ), + ); + + const entries = await listRegisteredRepos(); + expect(entries).toHaveLength(repoPaths.length); + expect(entries.map((entry) => entry.name).sort()).toEqual( + repoPaths.map((_, index) => `concurrent-${index}`).sort(), + ); + }); + it('re-registerRepo on same path without name preserves an existing alias', async () => { await registerRepo(tmpRepoA.dbPath, meta, { name: 'custom-alias' }); // Second call with no opts should keep the alias, not revert to basename. diff --git a/gitnexus/test/unit/watch-command.test.ts b/gitnexus/test/unit/watch-command.test.ts new file mode 100644 index 000000000..b59fbe33a --- /dev/null +++ b/gitnexus/test/unit/watch-command.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const autoSync = vi.hoisted(() => ({ + startAutoSyncWatch: vi.fn(), +})); + +vi.mock('../../src/core/auto-sync/index.js', () => ({ + getAutoSyncConfigPath: vi.fn(() => '/tmp/watch_config.yml'), + getAutoSyncMutexPath: vi.fn(() => '/tmp/watch.mutex'), + readAutoSyncWatchStatus: vi.fn(), + resetAutoSyncState: vi.fn(), + startAutoSyncWatch: autoSync.startAutoSyncWatch, + stopAutoSyncWatch: vi.fn(), +})); + +import { watchCommand } from '../../src/cli/watch.js'; + +describe('watch command', () => { + beforeEach(() => vi.clearAllMocks()); + afterEach(() => vi.restoreAllMocks()); + + it('reports foreground stop failures and exits non-zero', async () => { + const stop = vi.fn(async () => { + throw new Error('cleanup failed'); + }); + autoSync.startAutoSyncWatch.mockResolvedValue({ stop }); + let signalHandler: (() => void) | undefined; + vi.spyOn(process, 'once').mockImplementation(((event, listener) => { + if (event === 'SIGTERM') signalHandler = listener as () => void; + return process; + }) as typeof process.once); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + + await watchCommand('start'); + signalHandler?.(); + await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(1)); + + expect(stop).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith('[auto-sync] Failed to stop watch: cleanup failed\n'); + expect(stderr).not.toHaveBeenCalledWith('[auto-sync] Watch stopped.\n'); + }); +}); From 9bad9b8b2ec2e69404c0e7f7025396fbac844a24 Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 4 Aug 2026 17:36:25 +0800 Subject: [PATCH 09/14] update agents & claude md --- AGENTS.md | 14 +++++++------- CLAUDE.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2e03061e2..dba974980 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ mirror. `gitnexus/test/unit/shipped-skills-sync.test.ts` guards the copies. Toke # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **GitNexus** (22756 symbols, 58464 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). @@ -144,12 +144,12 @@ This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relati | Task | Read this skill file | |------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus-cli/SKILL.md` | +| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | diff --git a/CLAUDE.md b/CLAUDE.md index cede8bee9..9048032a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ See the ` … ` block in **[AGENTS.m # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **GitNexus** (22756 symbols, 58464 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). @@ -95,11 +95,11 @@ This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relati | Task | Read this skill file | |------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus-cli/SKILL.md` | +| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | From 3890639b210c8b35e00259eab84ae8dead0ce52d Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 4 Aug 2026 18:04:44 +0800 Subject: [PATCH 10/14] merge main --- gitnexus/src/storage/repo-manager.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index b98e4a680..3f0b16f68 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -22,7 +22,6 @@ import { randomBytes } from 'crypto'; import { getInferredRepoName, resolveRepoIdentityRoot } from './git.js'; import { stripWindowsLongPathPrefix } from '../lib/utils.js'; import { retryRename } from './fs-atomic.js'; -import { acquireFileLock } from './file-lock.js'; import { logger } from '../core/logger.js'; import type { UnresolvedReceiverSummary } from '../core/ingestion/scope-resolution/unresolved-receivers.js'; import { acquireIndexLock, IndexLockTimeoutError, type IndexLockHandle } from './index-lock.js'; @@ -1329,18 +1328,6 @@ const writeRegistry = async (entries: RegistryEntry[]): Promise => { } }; -const withRegistryLock = async (operation: () => Promise): Promise => { - const release = await acquireFileLock(`${getGlobalRegistryPath()}.lock`, { - retries: 400, - retryDelayMs: 25, - }); - try { - return await operation(); - } finally { - await release(); - } -}; - /** * Options for {@link registerRepo}. All optional — callers without any * disambiguation requirement can keep calling `registerRepo(path, meta)` @@ -2060,7 +2047,6 @@ export const listRegisteredRepos = async (opts?: { // Validate each entry still has a .gitnexus/ directory with metadata const valid: RegistryEntry[] = []; - const prunedPaths: string[] = []; for (const entry of entries) { // Named to avoid shadowing the exported `hasIndex` function above. let indexFound = false; @@ -2091,7 +2077,6 @@ export const listRegisteredRepos = async (opts?: { valid.push(entry); } else if (!firstNonMissingError && lastMissingError) { // Index genuinely removed — safe to prune - prunedPaths.push(canonicalizePath(entry.path)); } else { // Not provably absent — keep entry to prevent mass registry wipe. // Warn so an I/O storm becomes observable instead of silently From 5e92b1518fdffe878de3ee1f6eee8b49ebd25436 Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 25 Aug 2026 20:44:33 +0800 Subject: [PATCH 11/14] fix(watch): harden auto-sync lifecycle --- gitnexus/src/cli/i18n/en.ts | 2 +- gitnexus/src/cli/i18n/zh-CN.ts | 2 +- .../core/auto-sync/analysis-worker-launch.ts | 30 ++++++--- gitnexus/src/core/auto-sync/config.ts | 8 ++- gitnexus/src/core/auto-sync/path-security.ts | 3 +- gitnexus/src/core/auto-sync/runner.ts | 8 ++- gitnexus/src/server/analyze-worker-core.ts | 58 +++++++--------- gitnexus/src/server/analyze-worker.ts | 19 +----- gitnexus/src/server/git-clone.ts | 5 +- gitnexus/src/storage/file-lock.ts | 14 +++- gitnexus/src/storage/repo-manager.ts | 16 ++--- .../test/unit/analyze-worker-core.test.ts | 31 --------- .../unit/auto-sync-analysis-worker.test.ts | 67 ++++++++++++++++++- gitnexus/test/unit/auto-sync-runner.test.ts | 18 +++-- gitnexus/test/unit/auto-sync.test.ts | 26 +++++++ gitnexus/test/unit/file-lock.test.ts | 26 +++++++ gitnexus/test/unit/git-clone.test.ts | 23 +++++++ 17 files changed, 235 insertions(+), 121 deletions(-) diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 7984c9be3..8d8fc5fc1 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -135,7 +135,7 @@ export const en = { 'help.command.watch.description': 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', 'help.watch.details': - '\nActions: init, start (default), restart, stop, status, reset\nConfiguration: GITNEXUS_HOME/watch_config.yml\nRuntime files: GITNEXUS_HOME/watch/watch.pid, watch.mutex, watch.owner.json, watch.status.json, auto-sync-state.json\nRecovery: mutexes with verified dead owners are reclaimed automatically; invalid or legacy mutexes fail closed and require manual removal after confirming no watch process is running.\nWrites: GITNEXUS_HOME/watch/project_commit_info.txt\nRemote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.\nRuns once immediately, then repeats on sync_interval_minutes.', + '\nActions: init, start (default), restart, stop, status, reset\nConfiguration: GITNEXUS_HOME/watch_config.yml\nRuntime files: GITNEXUS_HOME/watch/watch.pid, watch.mutex, watch.owner.json, watch.status.json, auto-sync-state.json\nRecovery: mutexes with verified dead owners are reclaimed automatically; invalid or legacy mutexes fail closed and require manual removal after confirming no watch process is running.\nWrites: GITNEXUS_HOME/watch/project_commit_info.txt\nRemote URLs: only SSH URLs on github.com, gitlab.com, and gitee.com are allowed.\nRuns once immediately, then repeats on sync_interval_minutes.', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 758a4c2f3..aa6e06498 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -136,7 +136,7 @@ export const zhCN = { 'help.command.watch.description': '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', 'help.watch.details': - '\n操作：init、start（默认）、restart、stop、status、reset\n配置：GITNEXUS_HOME/watch_config.yml\n运行时文件：GITNEXUS_HOME/watch/watch.pid、watch.mutex、watch.owner.json、watch.status.json、auto-sync-state.json\n恢复：已验证 owner 退出的 mutex 会自动回收；无效或旧版 mutex 会安全拒绝，确认没有 watch 进程运行后再手动删除。\n写入：GITNEXUS_HOME/watch/project_commit_info.txt\n远程地址：仅允许 git@github.com:owner/repo.git、git@gitlab.com:group/repo.git 和 git@gitee.com:owner/repo.git。\n启动后立即运行一次，之后按 sync_interval_minutes 重复。', + '\n操作：init、start（默认）、restart、stop、status、reset\n配置：GITNEXUS_HOME/watch_config.yml\n运行时文件：GITNEXUS_HOME/watch/watch.pid、watch.mutex、watch.owner.json、watch.status.json、auto-sync-state.json\n恢复：已验证 owner 退出的 mutex 会自动回收；无效或旧版 mutex 会安全拒绝，确认没有 watch 进程运行后再手动删除。\n写入：GITNEXUS_HOME/watch/project_commit_info.txt\n远程地址：仅允许 github.com、gitlab.com 和 gitee.com 上的 SSH 地址。\n启动后立即运行一次，之后按 sync_interval_minutes 重复。', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index cda91902a..b92aaaa84 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -67,6 +67,7 @@ export function createAutoSyncAnalysisRunner( let terminalOutcome: WorkerMessage | undefined; let terminationGrace: ReturnType | undefined; + let terminationReason: 'timeout' | 'cancelled' | undefined; let settled = false; const cleanup = () => { deps.clearTimeoutFn(timeout); @@ -80,28 +81,37 @@ export function createAutoSyncAnalysisRunner( if (error) reject(error); else resolve(result!); }; - const timeout = deps.setTimeoutFn(() => { + const requestTermination = (reason: 'timeout' | 'cancelled') => { + if (settled || terminationReason) return; + terminationReason = reason; + deps.clearTimeoutFn(timeout); child.kill('SIGTERM'); terminationGrace = deps.setTimeoutFn(() => { child.kill('SIGKILL'); - settle(new Error(`Analysis timed out after ${timeoutMs}ms.`)); + settle( + new Error( + reason === 'timeout' ? `Analysis timed out after ${timeoutMs}ms.` : 'Analysis cancelled.', + ), + ); }, TERMINATION_GRACE_MS); - }, timeoutMs); - const onAbort = () => { - child.kill('SIGKILL'); - settle(new Error('Analysis cancelled.')); }; + const timeout = deps.setTimeoutFn(() => requestTermination('timeout'), timeoutMs); + const onAbort = () => requestTermination('cancelled'); signal?.addEventListener('abort', onAbort, { once: true }); child.on('message', (message: WorkerMessage) => { - if (message.type !== 'progress') terminalOutcome ??= message; + // Once timeout/cancellation requested shutdown, its reason owns the + // result. A terminal IPC can already be queued behind SIGTERM. + if (message.type === 'progress' || terminalOutcome || terminationReason) return; + terminalOutcome = message; + deps.clearTimeoutFn(timeout); }); child.on('error', (error) => { settle(new Error(`Auto-sync analyze worker error: ${error.message}`)); }); child.on('exit', (code, childSignal) => { if (settled) return; - if (terminationGrace) { + if (terminationReason === 'timeout') { settle( new Error( `Analysis timed out after ${timeoutMs}ms and worker exited (${childSignal ?? code ?? 'unknown'}).`, @@ -109,6 +119,10 @@ export function createAutoSyncAnalysisRunner( ); return; } + if (terminationReason === 'cancelled') { + settle(new Error('Analysis cancelled.')); + return; + } if (terminalOutcome?.type === 'complete') { settle(undefined, { stats: terminalOutcome.result.stats }); return; diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index f85c0b812..230df86c1 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -249,7 +249,7 @@ export function validateAutoSyncRemoteUrl(remoteUrl: string): void { const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim()); if (!match) { throw new Error( - 'must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git', + 'must use an SSH URL on github.com, gitlab.com, or gitee.com', ); } const host = match[1].toLowerCase(); @@ -270,6 +270,12 @@ export function validateAutoSyncBranchName(branch: string): void { if (branch.startsWith('-')) throw new Error('must not start with "-"'); if (branch.includes('..')) throw new Error('must not contain ".."'); if (branch.includes('`')) throw new Error('must not contain backticks'); + if (branch.endsWith('/') || branch.endsWith('.')) + throw new Error('must not end with "/" or "."'); + if (branch.includes('//')) throw new Error('must not contain consecutive slashes'); + if (branch.includes('@{')) throw new Error('must not contain "@{"'); + if (branch.split('/').some((component) => component.startsWith('.') || component.endsWith('.lock'))) + throw new Error('must not contain hidden or .lock path components'); } export function parseDurationMs(value: unknown): number { diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts index 2b1d3e77b..a782c2028 100644 --- a/gitnexus/src/core/auto-sync/path-security.ts +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -1,4 +1,5 @@ import fs from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; import os from 'node:os'; import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; @@ -115,7 +116,7 @@ export async function quarantineAutoSyncPartial( await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); const base = path.basename(targetDir); const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const destination = path.join(quarantineRoot, `auto-sync-${stamp}-${process.pid}-${base}`); + const destination = path.join(quarantineRoot, `auto-sync-${stamp}-${process.pid}-${randomUUID()}-${base}`); try { await fs.rename(targetDir, destination); } catch (err: unknown) { diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index 70d159eba..f0b148a46 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -301,8 +301,9 @@ export async function runAutoSyncOnce( if (repoResult.project.groupName) { let groupMembershipOk = false; + let membershipAdded = false; try { - await deps.addRepoToGroup( + membershipAdded = await deps.addRepoToGroup( repoResult.project, getAutoSyncRepoIdentity(repoResult.remoteUrl), getAutoSyncRepoIdentity(repoResult.remoteUrl), @@ -314,7 +315,10 @@ export async function runAutoSyncOnce( `[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`, ); } - if (groupMembershipOk && analyzeStatus === 'success') { + if ( + groupMembershipOk && + (analyzeStatus === 'success' || (membershipAdded && analyzeStatus === 'skipped')) + ) { groupsToSync.add(repoResult.project.groupName); } } diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index dcee256a4..916cce42c 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -27,7 +27,6 @@ import { IndexLockTimeoutError } from '../storage/index-lock.js'; export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; assertAnalysisFinalized: typeof import('../storage/repo-manager.js').assertAnalysisFinalized; - acquireAnalysisLock: (repoPath: string) => Promise<() => Promise>; send: (msg: WorkerMessage) => void; /** * Claim the single terminal-outcome slot. Returns `true` for the first caller @@ -52,38 +51,33 @@ export async function runWorkerAnalysis( ): Promise { let terminal: WorkerMessage; try { - const releaseAnalysisLock = await deps.acquireAnalysisLock(repoPath); - try { - const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap - ? [runnerIdentityAtBootstrap] - : []; - const result = await deps.runFullAnalysis( - repoPath, - // This worker force-exits right after reporting, so skip the native close - // (it can double-free in LadybugDB's ClientContext destructor after --pdg - // writes); flushWAL still persists the index, process.exit reclaims handles. - { ...options, skipNativeCloseOnExit: true }, - { - onProgress: (phase, percent, message) => - deps.send({ type: 'progress', phase, percent, message }), - onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), - }, - ...bootstrapArgs, - ); - // P2 (#2264): a half-finalized repo — meta.json written but the global - // registry entry missing (e.g. a prior collision-aborted run, or a wiped - // registry) — must NOT be reported as a successful analysis. Mirror the CLI's - // assertAnalysisFinalized guard so the worker surfaces it as an error instead - // of a false `complete` that leaves the repo invisible to list_repos. - await deps.assertAnalysisFinalized(repoPath); + const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap + ? [runnerIdentityAtBootstrap] + : []; + const result = await deps.runFullAnalysis( + repoPath, + // This worker force-exits right after reporting, so skip the native close + // (it can double-free in LadybugDB's ClientContext destructor after --pdg + // writes); flushWAL still persists the index, process.exit reclaims handles. + { ...options, skipNativeCloseOnExit: true }, + { + onProgress: (phase, percent, message) => + deps.send({ type: 'progress', phase, percent, message }), + onLog: (message) => deps.send({ type: 'progress', phase: 'log', percent: -1, message }), + }, + ...bootstrapArgs, + ); + // P2 (#2264): a half-finalized repo — meta.json written but the global + // registry entry missing (e.g. a prior collision-aborted run, or a wiped + // registry) — must NOT be reported as a successful analysis. Mirror the CLI's + // assertAnalysisFinalized guard so the worker surfaces it as an error instead + // of a false `complete` that leaves the repo invisible to list_repos. + await deps.assertAnalysisFinalized(repoPath); - // Send a JSON-safe projection, NOT the raw result: the IPC channel is - // default-JSON serialization and `result.pipelineResult` carries the live - // KnowledgeGraph. See analyze-worker-ipc.ts. - terminal = { type: 'complete', result: projectAnalyzeResultForIpc(result) }; - } finally { - await releaseAnalysisLock(); - } + // Send a JSON-safe projection, NOT the raw result: the IPC channel is + // default-JSON serialization and `result.pipelineResult` carries the live + // KnowledgeGraph. See analyze-worker-ipc.ts. + terminal = { type: 'complete', result: projectAnalyzeResultForIpc(result) }; } catch (err: unknown) { // Report the failure to the parent over IPC (the parent surfaces the message). const message = err instanceof Error ? err.message : 'Analysis failed'; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index b78699257..ad4632eb0 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -11,8 +11,6 @@ * Child -> Parent: { type: 'error', message: string } */ -import path from 'path'; -import { createHash } from 'crypto'; import type { StartMessage, WorkerMessage } from './analyze-worker-protocol.js'; import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js'; type BoundedCheckpointBeforeExit = @@ -105,13 +103,12 @@ process.on('message', async (msg: StartMessage) => { const prepared = await identityModule.captureAnalyzerIdentityBeforeLoad( import.meta.url, async () => { - const [analysisModule, repoManager, shutdownHelpers, fileLock] = await Promise.all([ + const [analysisModule, repoManager, shutdownHelpers] = await Promise.all([ import('../core/run-analyze.js'), import('../storage/repo-manager.js'), import('../core/lbug/shutdown-helpers.js'), - import('../storage/file-lock.js'), ]); - return { analysisModule, repoManager, shutdownHelpers, fileLock }; + return { analysisModule, repoManager, shutdownHelpers }; }, ); boundedCheckpointBeforeExit = prepared.loaded.shutdownHelpers.boundedCheckpointBeforeExit; @@ -125,18 +122,6 @@ process.on('message', async (msg: StartMessage) => { { runFullAnalysis: prepared.loaded.analysisModule.runFullAnalysis, assertAnalysisFinalized: prepared.loaded.repoManager.assertAnalysisFinalized, - acquireAnalysisLock: (repoPath) => { - const repoKey = createHash('sha256') - .update(prepared.loaded.repoManager.canonicalizePath(repoPath)) - .digest('hex'); - return prepared.loaded.fileLock.acquireFileLock( - path.join( - prepared.loaded.repoManager.getGlobalDir(), - 'locks', - `analyze-${repoKey}.lock`, - ), - ); - }, send, claimTerminal, }, diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 71c6a98a7..7598068ad 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -523,6 +523,8 @@ export async function cloneOrPull( await assertDirectoryOwnerAndPermissions(cloneRoot); } await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); + await fs.mkdir(path.dirname(safeTarget), { recursive: true }); + await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); await assertPreRealpathContainment(cloneRoot, safeTarget); const exists = await fs.access(path.join(safeTarget, '.git')).then( @@ -600,9 +602,6 @@ export async function cloneOrPull( if (targetExists && (await fs.readdir(safeTarget)).length > 0) { throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); } - await fs.mkdir(path.dirname(safeTarget), { recursive: true }); - await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); - await assertPreRealpathContainment(cloneRoot, safeTarget); onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); try { const runGitImpl = options?.runGitForTest ?? runGit; diff --git a/gitnexus/src/storage/file-lock.ts b/gitnexus/src/storage/file-lock.ts index 2ed6a66cc..b509172b7 100644 --- a/gitnexus/src/storage/file-lock.ts +++ b/gitnexus/src/storage/file-lock.ts @@ -61,6 +61,7 @@ export async function acquireFileLock( if ( await reclaimStaleLock( resolvedPath, + owner, options.isProcessAlive ?? isProcessAlive, options.readProcessStartTime ?? readProcessStartTime, ) @@ -81,14 +82,21 @@ export async function acquireFileLock( async function reclaimStaleLock( lockPath: string, + guardOwner: FileLockOwner, ownerIsAlive: (pid: number) => boolean, getProcessStartTime: (pid: number) => string | undefined, ): Promise { const reclaimGuardPath = `${lockPath}.reclaim`; + let releaseReclaimGuard: () => Promise; try { - await fs.mkdir(reclaimGuardPath); + releaseReclaimGuard = await acquireFileLock(reclaimGuardPath, { + pid: guardOwner.pid, + processStartTime: guardOwner.processStartTime, + isProcessAlive: ownerIsAlive, + readProcessStartTime: getProcessStartTime, + }); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'EEXIST') return false; + if (error instanceof FileLockBusyError) return false; throw error; } @@ -103,7 +111,7 @@ async function reclaimStaleLock( await fs.rm(lockPath, { force: true }); return true; } finally { - await fs.rmdir(reclaimGuardPath); + await releaseReclaimGuard(); } } diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index e1630b388..9c302a013 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -18,7 +18,6 @@ import fs from 'fs/promises'; import { realpathSync } from 'fs'; import path from 'path'; import os from 'os'; -import { randomBytes } from 'crypto'; import { getInferredRepoName, resolveRepoIdentityRoot, stripUrlCredentials } from './git.js'; import { stripWindowsLongPathPrefix } from '../lib/utils.js'; import { writeFileAtomic } from './fs-atomic.js'; @@ -642,17 +641,10 @@ export const readRegistry = async (): Promise => { const writeRegistry = async (entries: RegistryEntry[]): Promise => { const dir = getGlobalDir(); await fs.mkdir(dir, { recursive: true }); - // Atomic tmp+rename (mirrors saveMeta): a crash mid-write can never leave a - // truncated/half-written registry.json that the next load would treat as - // empty and silently drop every registered repo (#2106 R9). - const target = getGlobalRegistryPath(); - const tmp = `${target}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; - try { - await fs.writeFile(tmp, JSON.stringify(sanitizeEntries(entries), null, 2), 'utf-8'); - await fs.rename(tmp, target); - } finally { - await fs.unlink(tmp).catch(() => {}); - } + await writeFileAtomic( + getGlobalRegistryPath(), + JSON.stringify(sanitizeEntries(entries), null, 2), + ); }; /** diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index 2309d34ea..06e3acd35 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -32,7 +32,6 @@ const baseResult: AnalyzeResult = { const okRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => baseResult); const okFinalize: WorkerAnalysisDeps['assertAnalysisFinalized'] = vi.fn(async () => undefined); -const okLock: WorkerAnalysisDeps['acquireAnalysisLock'] = vi.fn(async () => async () => undefined); const alwaysClaim: WorkerAnalysisDeps['claimTerminal'] = () => true; describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { @@ -50,7 +49,6 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized, - acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -72,7 +70,6 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized: okFinalize, - acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -93,7 +90,6 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: run, assertAnalysisFinalized: okFinalize, - acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -103,31 +99,6 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { expect(run.mock.calls[0]?.[3]).toBe(receipt); }); - it('does not enter analysis when another worker holds the repo lock', async () => { - const send = vi.fn<(msg: WorkerMessage) => void>(); - const run = vi.fn(async () => baseResult); - - await runWorkerAnalysis( - '/repo', - {}, - { - runFullAnalysis: run, - assertAnalysisFinalized: okFinalize, - acquireAnalysisLock: vi.fn(async () => { - throw new Error('Lock is already held for /repo'); - }), - send, - claimTerminal: alwaysClaim, - }, - ); - - expect(run).not.toHaveBeenCalled(); - expect(send).toHaveBeenCalledWith({ - type: 'error', - message: 'Lock is already held for /repo', - }); - }); - it('reports error when finalization passes but the analysis itself throws', async () => { const send = vi.fn<(msg: WorkerMessage) => void>(); const failingRun: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => { @@ -143,7 +114,6 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { { runFullAnalysis: failingRun, assertAnalysisFinalized: finalize, - acquireAnalysisLock: okLock, send, claimTerminal: alwaysClaim, }, @@ -196,7 +166,6 @@ describe('runWorkerAnalysis — terminal-claim coordination (#2264 P3)', () => { { runFullAnalysis: okRun, assertAnalysisFinalized: okFinalize, - acquireAnalysisLock: okLock, send, claimTerminal: alreadyClaimed, }, diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 2908bfc32..63a78ade4 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -80,7 +80,67 @@ describe('auto-sync analysis worker', () => { await expect(result).rejects.toThrow('Analysis timed out after 50ms'); }); - it('kills an active worker immediately when watch is stopped', async () => { + it('keeps the timeout outcome when complete arrives after termination begins', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + timers[0](); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).rejects.toThrow('Analysis timed out after 50ms'); + }); + + it('clears the analysis deadline after complete before the worker exits', async () => { + vi.useFakeTimers(); + try { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + await vi.advanceTimersByTimeAsync(50); + + expect(child.kill).not.toHaveBeenCalled(); + child.emit('exit', 0, null); + await expect(result).resolves.toEqual({ stats: { files: 3 } }); + } finally { + vi.useRealTimers(); + } + }); + + it('keeps the cancellation outcome when complete arrives after abort', async () => { + const child = Object.assign(new EventEmitter(), { + send: vi.fn(), + kill: vi.fn(), + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + const controller = new AbortController(); + + const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); + controller.abort(); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).rejects.toThrow('Analysis cancelled'); + }); + + it('asks an active worker to stop gracefully when watch is stopped', async () => { const child = Object.assign(new EventEmitter(), { send: vi.fn(), kill: vi.fn(), @@ -93,8 +153,9 @@ describe('auto-sync analysis worker', () => { const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); controller.abort(); - expect(child.kill).toHaveBeenCalledWith('SIGKILL'); - child.emit('exit', null, 'SIGKILL'); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + expect(child.kill).not.toHaveBeenCalledWith('SIGKILL'); + child.emit('exit', null, 'SIGTERM'); await expect(result).rejects.toThrow('Analysis cancelled'); }); }); diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 8a924c877..2bffaed9b 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -295,7 +295,7 @@ describe('auto-sync runner', () => { })), saveState: vi.fn(async () => {}), writeCommitInfo: vi.fn(async () => {}), - addRepoToGroup: vi.fn(async () => false), + addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), getAvailableMemoryGB: vi.fn(() => 8), }); @@ -308,7 +308,7 @@ describe('auto-sync runner', () => { expect(result.analyzed).toBe(0); expect(result.skippedAnalysis).toBe(1); expect(deps.runAnalysis).not.toHaveBeenCalled(); - expect(deps.syncGroupByName).not.toHaveBeenCalled(); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); }); it('uses remote identity under local_path as the clone target', async () => { @@ -1115,13 +1115,14 @@ describe('auto-sync starter', () => { return timer; }) as unknown as typeof setInterval; const stderr = { write: vi.fn() }; - let releaseRun: (() => void) | undefined; + const releaseRuns: Array<() => void> = []; const runOnce = vi.fn( () => new Promise((resolve) => { - releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + releaseRuns.push(() => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); }), ); + let handle: Awaited> | undefined; try { process.env.GITNEXUS_HOME = tempDir; @@ -1137,7 +1138,7 @@ describe('auto-sync starter', () => { ].join('\n'), ); - await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); + handle = await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); scheduled?.(); expect(runOnce).toHaveBeenCalledTimes(1); @@ -1145,12 +1146,17 @@ describe('auto-sync starter', () => { '[auto-sync] Previous run is still active; skipping overlapping run.\n', ); - releaseRun?.(); + releaseRuns.shift()?.(); await new Promise((resolve) => setTimeout(resolve, 0)); scheduled?.(); expect(runOnce).toHaveBeenCalledTimes(2); + releaseRuns.shift()?.(); + await handle?.stop(); + handle = undefined; } finally { + releaseRuns.splice(0).forEach((release) => release()); + await handle?.stop(); if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; await fs.rm(tempDir, { recursive: true, force: true }); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 940f23871..00ac48ded 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -351,6 +351,24 @@ describe('auto-sync', () => { await expect(fs.access(target)).rejects.toThrow(); }); + it('gives concurrent partial clone quarantines unique destinations', async () => { + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + const first = path.join(tempDir, 'one', 'partial-repo'); + const second = path.join(tempDir, 'two', 'partial-repo'); + await Promise.all([fs.mkdir(first, { recursive: true }), fs.mkdir(second, { recursive: true })]); + + const [firstDestination, secondDestination] = await Promise.all([ + quarantineAutoSyncPartial(first, quarantineRoot), + quarantineAutoSyncPartial(second, quarantineRoot), + ]); + + expect(firstDestination).not.toBe(secondDestination); + await expect(fs.access(firstDestination)).resolves.toBeUndefined(); + await expect(fs.access(secondDestination)).resolves.toBeUndefined(); + await expect(fs.access(first)).rejects.toThrow(); + await expect(fs.access(second)).rejects.toThrow(); + }); + it('rejects group-writable configured clone roots', async () => { if (process.platform === 'win32') return; const root = path.join(tempDir, 'group-writable-repos'); @@ -388,10 +406,17 @@ describe('auto-sync', () => { it('rejects unsafe auto-sync branch names', () => { expect(() => validateAutoSyncBranchName('feature/good-branch')).not.toThrow(); + expect(() => validateAutoSyncBranchName('foo./bar')).not.toThrow(); expect(() => validateAutoSyncBranchName('-upload-pack=evil')).toThrow('must not start'); expect(() => validateAutoSyncBranchName('feature bad')).toThrow('whitespace'); expect(() => validateAutoSyncBranchName('feature..bad')).toThrow('must not contain ".."'); expect(() => validateAutoSyncBranchName('bad:ref')).toThrow('not allowed'); + expect(() => validateAutoSyncBranchName('feature.')).toThrow('must not end'); + expect(() => validateAutoSyncBranchName('feature/')).toThrow('must not end'); + expect(() => validateAutoSyncBranchName('feature//branch')).toThrow('consecutive'); + expect(() => validateAutoSyncBranchName('feature@{x')).toThrow('must not contain "@{"'); + expect(() => validateAutoSyncBranchName('.hidden')).toThrow('hidden'); + expect(() => validateAutoSyncBranchName('foo/bar.lock')).toThrow('hidden or .lock'); }); it('extracts safe repository names from remote URLs', () => { @@ -411,6 +436,7 @@ describe('auto-sync', () => { }); it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => { + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo')).not.toThrow(); expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow(); expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow(); expect(() => diff --git a/gitnexus/test/unit/file-lock.test.ts b/gitnexus/test/unit/file-lock.test.ts index b02704b10..aba2a2ac5 100644 --- a/gitnexus/test/unit/file-lock.test.ts +++ b/gitnexus/test/unit/file-lock.test.ts @@ -65,6 +65,15 @@ describe('file lock', () => { const lockPath = await tempLockPath(); await acquireFileLock(lockPath, { pid: 111, processStartTime: 'old-start' }); + await expect( + acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'next-start', + isProcessAlive: () => true, + readProcessStartTime: () => 'old-start', + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + const nextRelease = await acquireFileLock(lockPath, { pid: 222, processStartTime: 'next-start', @@ -83,6 +92,23 @@ describe('file lock', () => { await expect(fs.access(lockPath)).resolves.toBeUndefined(); }); + it('recovers when a stale reclaim guard was left by a crashed contender', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 999, processStartTime: 'abandoned' }); + await acquireFileLock(`${lockPath}.reclaim`, { + pid: 998, + processStartTime: 'abandoned-reclaimer', + }); + + const release = await acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }); + + await release(); + }); + it('waits for the current holder when retries are configured', async () => { const lockPath = await tempLockPath(); const release = await acquireFileLock(lockPath); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index ac3d3b396..cb05792ae 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -637,6 +637,29 @@ describe('git-clone', () => { } }); + it('creates missing nested parents before checking controlled clone containment', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'github.com', 'owner', 'repo'); + const runGitForTest = vi.fn(async () => { + await fs.mkdir(path.join(target, '.git'), { recursive: true }); + return ''; + }); + try { + await expect( + cloneOrPull('git@github.com:owner/repo', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + runGitForTest, + }), + ).resolves.toBe(target); + + expect(runGitForTest).toHaveBeenCalledOnce(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('allows auto-sync SSH SCP clone URLs with a per-repo timeout', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); const target = path.join(root, 'repo'); From 3b590acc792544e4574a1eae939cbd7398727d46 Mon Sep 17 00:00:00 2001 From: weiyf Date: Wed, 26 Aug 2026 16:52:18 +0800 Subject: [PATCH 12/14] fix(watch): normalize SSH repo identity paths --- gitnexus/src/core/auto-sync/runner.ts | 4 +-- gitnexus/test/unit/auto-sync-runner.test.ts | 36 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index f0b148a46..b5c09756f 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -223,7 +223,7 @@ export async function runAutoSyncOnce( kind: 'failed' as const, project: item.project, remoteUrl: item.remoteUrl, - targetDir: '', + targetDir: item.targetDir ?? '', status: 'sync_failed' as const, lastSyncTime, }; @@ -372,7 +372,7 @@ export async function addRepoToGroup( export function getAutoSyncRepoIdentity(remoteUrl: string): string { validateAutoSyncRemoteUrl(remoteUrl); const [, host, remotePath] = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim())!; - return `${host.toLowerCase()}/${remotePath.replace(/\.git$/, '')}`; + return `${host.toLowerCase()}/${remotePath.replace(/\.git$/i, '')}`; } export async function syncGroupByName(groupName: string): Promise { diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 2bffaed9b..b5a1e4d50 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from 'vitest'; import { addRepoToGroup, + getAutoSyncRepoIdentity, getConfiguredRepoPath, getAutoSyncWatchPaths, readAutoSyncWatchStatus, @@ -278,6 +279,12 @@ describe('auto-sync runner', () => { ); }); + it('normalizes the .git suffix case in auto-sync repository identities', () => { + expect(getAutoSyncRepoIdentity('git@GitHub.com:team/service.GIT')).toBe( + 'github.com/team/service', + ); + }); + it('skips analysis when commit id has not changed', async () => { const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), @@ -619,6 +626,35 @@ describe('auto-sync runner', () => { ); }); + it('records the resolved target directory when a post-sync operation fails', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => { + throw new Error('git log failed'); + }), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).resolves.toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + localPath: '/tmp/repos/gitee.com/qts_server/qts_account', + status: 'sync_failed', + }), + ]); + }); + it('isolates clone-root resolution failures to the affected project', async () => { const isolatedConfig: AutoSyncConfig = { ...config, From e1d8c65f8c02eb343f8e079a3750587448aa96bd Mon Sep 17 00:00:00 2001 From: weiyf Date: Wed, 26 Aug 2026 16:53:09 +0800 Subject: [PATCH 13/14] fix(watch): normalize SSH repo identity paths --- .../core/auto-sync/analysis-worker-launch.ts | 10 ++++++++-- gitnexus/src/core/auto-sync/config.ts | 6 +++++- gitnexus/src/server/git-clone.ts | 4 ++++ .../unit/auto-sync-analysis-worker.test.ts | 13 ++++++++++++- gitnexus/test/unit/auto-sync.test.ts | 6 ++++++ gitnexus/test/unit/file-lock.test.ts | 19 ++++++++++++++++--- gitnexus/test/unit/git-clone.test.ts | 9 +++++++++ 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index b92aaaa84..29060b85e 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; import type { WorkerMessage } from '../../server/analyze-worker.js'; +import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; const _require = createRequire(import.meta.url); const TERMINATION_GRACE_MS = 10_000; @@ -58,9 +59,14 @@ export function createAutoSyncAnalysisRunner( reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); return; } + const workerHeapMb = Math.min(8192, autoHeapCapMb()); const execArgv = isDev - ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href, '--max-old-space-size=8192'] - : ['--max-old-space-size=8192']; + ? [ + '--import', + pathToFileURL(_require.resolve('tsx/esm')).href, + `--max-old-space-size=${workerHeapMb}`, + ] + : [`--max-old-space-size=${workerHeapMb}`]; const child = deps.forkWorker(workerPath, execArgv); child.stdout?.resume(); child.stderr?.resume(); diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 230df86c1..02bd5a7c1 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -246,7 +246,11 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy } export function validateAutoSyncRemoteUrl(remoteUrl: string): void { - const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim()); + const trimmed = remoteUrl.trim(); + if (trimmed.includes('?') || trimmed.includes('#')) { + throw new Error('must not include query strings or fragments'); + } + const match = /^git@([^:\s/]+):([^\s]+)$/.exec(trimmed); if (!match) { throw new Error( 'must use an SSH URL on github.com, gitlab.com, or gitee.com', diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 7598068ad..e902b7061 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -117,6 +117,10 @@ export function validateGitUrl(url: string): void { throw new Error('Only https:// and http:// git URLs are allowed'); } + if (parsed.search || parsed.hash) { + throw new Error('Git URLs must not include query strings or fragments'); + } + const host = parsed.hostname.toLowerCase(); // Block known dangerous hostnames (cloud metadata services) diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index 63a78ade4..d6a7b42d6 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -1,5 +1,11 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, vi } from 'vitest'; + +const { autoHeapCapMbMock } = vi.hoisted(() => ({ autoHeapCapMbMock: vi.fn(() => 512) })); +vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ + autoHeapCapMb: autoHeapCapMbMock, +})); + import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; describe('auto-sync analysis worker', () => { @@ -10,9 +16,14 @@ describe('auto-sync analysis worker', () => { stdout: { resume: vi.fn() }, stderr: { resume: vi.fn() }, }); - const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + const forkWorker = vi.fn(() => child as any); + const run = createAutoSyncAnalysisRunner({ forkWorker }); const result = run('/tmp/repo', { branch: 'main' }, 50); + expect(forkWorker).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['--max-old-space-size=512']), + ); child.emit('message', { type: 'progress', phase: 'parsing', progress: 20 }); child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); child.emit('exit', 0, null); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 00ac48ded..668671761 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -452,6 +452,12 @@ describe('auto-sync', () => { expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow( 'host must be', ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo.git?ref=main')).toThrow( + 'must not include query strings or fragments', + ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo.git#main')).toThrow( + 'must not include query strings or fragments', + ); }); it('parses repo git timeout durations', () => { diff --git a/gitnexus/test/unit/file-lock.test.ts b/gitnexus/test/unit/file-lock.test.ts index aba2a2ac5..8c2a41abd 100644 --- a/gitnexus/test/unit/file-lock.test.ts +++ b/gitnexus/test/unit/file-lock.test.ts @@ -84,11 +84,24 @@ describe('file lock', () => { await nextRelease(); }); - it('fails closed for a legacy or invalid lock without owner metadata', async () => { + it('fails closed for legacy or invalid lock contents without owner metadata', async () => { const lockPath = await tempLockPath(); - await fs.mkdir(lockPath, { recursive: true }); + const invalidContents = ['legacy lock', '{not json', JSON.stringify({ pid: 123 })]; + await fs.mkdir(path.dirname(lockPath), { recursive: true }); - await expect(acquireFileLock(lockPath)).rejects.toBeInstanceOf(FileLockBusyError); + for (const content of invalidContents) { + await fs.writeFile(lockPath, content, 'utf-8'); + await expect( + acquireFileLock(lockPath, { pid: 456, processStartTime: 'next-start' }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.readFile(lockPath, 'utf-8')).resolves.toBe(content); + await fs.rm(lockPath); + } + + await fs.mkdir(lockPath, { recursive: true }); + await expect( + acquireFileLock(lockPath, { pid: 456, processStartTime: 'next-start' }), + ).rejects.toBeInstanceOf(FileLockBusyError); await expect(fs.access(lockPath)).resolves.toBeUndefined(); }); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index cb05792ae..0c365e359 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -197,6 +197,15 @@ describe('git-clone', () => { expect(() => validateGitUrl('http://gitlab.com/user/repo.git')).not.toThrow(); }); + it('rejects query strings and fragments instead of reinterpreting clone remotes', () => { + expect(() => validateGitUrl('https://github.com/user/repo.git?ref=main')).toThrow( + 'must not include query strings or fragments', + ); + expect(() => validateGitUrl('https://github.com/user/repo.git#main')).toThrow( + 'must not include query strings or fragments', + ); + }); + it('blocks SSH protocol', () => { expect(() => validateGitUrl('ssh://git@github.com/user/repo.git')).toThrow( 'Only https:// and http://', From fc2c8acbd2514887a36e99c6849267b8682afe83 Mon Sep 17 00:00:00 2001 From: weiyf Date: Thu, 27 Aug 2026 14:21:59 +0800 Subject: [PATCH 14/14] fix(watch): safely cancel analysis across platforms --- README.md | 2 +- gitnexus/README.md | 4 +- .../core/auto-sync/analysis-worker-launch.ts | 67 ++- gitnexus/src/core/auto-sync/config.ts | 8 +- gitnexus/src/core/auto-sync/runner.ts | 23 +- gitnexus/src/core/auto-sync/starter.ts | 172 +++++-- .../src/server/analyze-worker-protocol.ts | 9 +- gitnexus/src/server/analyze-worker.ts | 61 ++- gitnexus/src/storage/file-lock.ts | 13 +- gitnexus/src/storage/repo-manager.ts | 5 +- .../unit/auto-sync-analysis-worker.test.ts | 164 +++---- gitnexus/test/unit/auto-sync-runner.test.ts | 453 +++++++----------- gitnexus/test/unit/auto-sync.test.ts | 4 + gitnexus/test/unit/file-lock.test.ts | 14 +- ...repo-manager-registry-atomic-write.test.ts | 4 +- 15 files changed, 509 insertions(+), 494 deletions(-) diff --git a/README.md b/README.md index 93a61c71f..0cad8089e 100644 --- a/README.md +++ b/README.md @@ -477,7 +477,7 @@ projects: - `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. Clones are stored below it as `host/namespace/repo`. - Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. - `branches` are tried in order. The legacy `branch` field is supported, but do not set both. -- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. Timed-out workers are terminated before scheduling resumes. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten. Stopping watch cancels an active analysis immediately. +- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. Timeout and `watch stop` request safe cancellation; a worker in native work exits after reaching a JS-visible safe point. Until then, watch reports `cancelling` or `stopping` and retains ownership so another watch cannot take over. This behavior is the same on macOS and Windows. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten. - Add `group_name` only after creating that group with `gitnexus group create `. Partial clone output is isolated and removed after 14 days. See the [full watch configuration and runtime reference](gitnexus/README.md#gitnexus-watch) for concurrency, timeouts, failure thresholds, and runtime files. diff --git a/gitnexus/README.md b/gitnexus/README.md index 1d247df62..e163bed7a 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -284,7 +284,7 @@ gitnexus group impact --target --repo # Cross-repo ### `gitnexus watch` -`gitnexus watch` is the explicit long-running auto-sync entrypoint. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus watch init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, `status`, and `reset` manage the same `GITNEXUS_HOME` instance. `reset` removes only the derived analysis state and commit snapshot; clones, indexes, and registry entries are untouched. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.mutex` prevents multiple watch processes for one home, `watch.owner.json` records ownership metadata, `watch.pid` plus `watch.status.json` expose process state, and `quarantine/` stores partial clone output before entries are removed after 14 days. Mutexes with verified dead owners are reclaimed automatically after an abnormal exit. Invalid or legacy mutexes fail closed; confirm no watch process is running before manually removing `watch.mutex` and stale `watch.pid` / `watch.owner.json`. +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus watch init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, `status`, and `reset` manage the same `GITNEXUS_HOME` instance. `reset` removes only the derived analysis state and commit snapshot; clones, indexes, and registry entries are untouched. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.mutex` prevents multiple watch processes for one home, `watch.owner.json` records ownership metadata, `watch.pid` plus `watch.status.json` expose process state, `watch.stop..json` is a temporary owner-fenced stop request, and `quarantine/` stores partial clone output before entries are removed after 14 days. Mutexes with verified dead owners are reclaimed automatically after an abnormal exit. Invalid or legacy mutexes fail closed; confirm no watch process is running before manually removing `watch.mutex` and stale `watch.pid` / `watch.owner.json`. ```yaml sync_interval_minutes: 10 @@ -302,7 +302,7 @@ projects: - git@gitee.com:owner/repo.git ``` -`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal; each remote is cloned below it as `host/namespace/repo`, preventing same-basename repositories from colliding. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. On timeout watch terminates that worker, records the failed attempt, and resumes scheduling only after the worker exits. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes. Stopping watch cancels an active analysis worker immediately. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit or `gitnexus watch reset` clears the block and allows analysis again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal; each remote is cloned below it as `host/namespace/repo`, preventing same-basename repositories from colliding. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. Timeout and `watch stop` request safe cancellation; a worker already in native work exits after it returns to a JS-visible safe point. While waiting, watch reports `cancelling` or `stopping` and keeps its ownership files so another watch cannot take over. `watch stop` uses this same control path on macOS and Windows. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit or `gitnexus watch reset` clears the block and allows analysis again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts index 29060b85e..4ecd73653 100644 --- a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -4,20 +4,19 @@ import { createRequire } from 'node:module'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; -import type { WorkerMessage } from '../../server/analyze-worker.js'; +import type { WorkerMessage } from '../../server/analyze-worker-protocol.js'; import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; const _require = createRequire(import.meta.url); -const TERMINATION_GRACE_MS = 10_000; - export type AutoSyncAnalysisRunner = ( repoPath: string, options: AnalyzeOptions, timeoutMs: number, signal?: AbortSignal, + onCancellationRequested?: () => void, ) => Promise>; -interface AnalysisWorker extends Pick { +interface AnalysisWorker extends Pick { stdout?: Pick | null; stderr?: Pick | null; } @@ -42,7 +41,7 @@ export function createAutoSyncAnalysisRunner( overrides: Partial = {}, ): AutoSyncAnalysisRunner { const deps = { ...DEFAULT_DEPS, ...overrides }; - return (repoPath, options, timeoutMs, signal) => + return (repoPath, options, timeoutMs, signal, onCancellationRequested) => new Promise>((resolve, reject) => { if (signal?.aborted) { reject(new Error('Analysis cancelled.')); @@ -72,12 +71,10 @@ export function createAutoSyncAnalysisRunner( child.stderr?.resume(); let terminalOutcome: WorkerMessage | undefined; - let terminationGrace: ReturnType | undefined; - let terminationReason: 'timeout' | 'cancelled' | undefined; + let terminationError: Error | undefined; let settled = false; const cleanup = () => { deps.clearTimeoutFn(timeout); - if (terminationGrace) deps.clearTimeoutFn(terminationGrace); signal?.removeEventListener('abort', onAbort); }; const settle = (error?: Error, result?: Pick) => { @@ -87,46 +84,41 @@ export function createAutoSyncAnalysisRunner( if (error) reject(error); else resolve(result!); }; - const requestTermination = (reason: 'timeout' | 'cancelled') => { - if (settled || terminationReason) return; - terminationReason = reason; + const requestCancellation = (error: Error) => { + if (settled || terminationError) return; + terminationError = error; deps.clearTimeoutFn(timeout); - child.kill('SIGTERM'); - terminationGrace = deps.setTimeoutFn(() => { - child.kill('SIGKILL'); - settle( - new Error( - reason === 'timeout' ? `Analysis timed out after ${timeoutMs}ms.` : 'Analysis cancelled.', - ), - ); - }, TERMINATION_GRACE_MS); + onCancellationRequested?.(); + // IPC has the same semantics on macOS and Windows. The worker exits only + // after reaching a JS-visible safe point; this parent keeps ownership until then. + try { + child.send({ type: 'cancel' }); + } catch { + // A closed IPC channel still has an exit/error path. Do not force-kill a + // worker that may be inside native code. + } }; - const timeout = deps.setTimeoutFn(() => requestTermination('timeout'), timeoutMs); - const onAbort = () => requestTermination('cancelled'); + const timeout = deps.setTimeoutFn( + () => requestCancellation(new Error(`Analysis timed out after ${timeoutMs}ms.`)), + timeoutMs, + ); + const onAbort = () => requestCancellation(new Error('Analysis cancelled.')); signal?.addEventListener('abort', onAbort, { once: true }); child.on('message', (message: WorkerMessage) => { // Once timeout/cancellation requested shutdown, its reason owns the - // result. A terminal IPC can already be queued behind SIGTERM. - if (message.type === 'progress' || terminalOutcome || terminationReason) return; + // result. A terminal IPC can already be queued behind cancellation. + if (message.type === 'progress' || terminalOutcome || terminationError) return; terminalOutcome = message; deps.clearTimeoutFn(timeout); }); child.on('error', (error) => { - settle(new Error(`Auto-sync analyze worker error: ${error.message}`)); + requestCancellation(new Error(`Auto-sync analyze worker error: ${error.message}`)); }); child.on('exit', (code, childSignal) => { if (settled) return; - if (terminationReason === 'timeout') { - settle( - new Error( - `Analysis timed out after ${timeoutMs}ms and worker exited (${childSignal ?? code ?? 'unknown'}).`, - ), - ); - return; - } - if (terminationReason === 'cancelled') { - settle(new Error('Analysis cancelled.')); + if (terminationError) { + settle(terminationError); return; } if (terminalOutcome?.type === 'complete') { @@ -146,8 +138,9 @@ export function createAutoSyncAnalysisRunner( try { child.send({ type: 'start', repoPath, options }); } catch (error) { - child.kill('SIGKILL'); - settle(new Error(`Failed to start auto-sync analyze worker: ${(error as Error).message}`)); + requestCancellation( + new Error(`Failed to start auto-sync analyze worker: ${(error as Error).message}`), + ); } }); } diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 02bd5a7c1..681adc486 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -261,7 +261,13 @@ export function validateAutoSyncRemoteUrl(remoteUrl: string): void { if (!ALLOWED_REMOTE_HOSTS.has(host)) { throw new Error('host must be one of github.com, gitlab.com, or gitee.com'); } - if (repoPath.startsWith('/') || repoPath.includes('..') || repoPath.split('/').length < 2) { + const pathParts = repoPath.split('/'); + if ( + repoPath.startsWith('/') || + repoPath.includes('..') || + pathParts.length < 2 || + pathParts.some((part) => !part) + ) { throw new Error('path must include owner/repo without traversal'); } } diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts index b5c09756f..ce3d82acd 100644 --- a/gitnexus/src/core/auto-sync/runner.ts +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -85,6 +85,7 @@ export async function runAutoSyncOnce( logger?: AutoSyncLogger; now?: () => Date; signal?: AbortSignal; + onAnalysisCancellationRequested?: () => void; } = {}, ): Promise { const deps = { ...DEFAULT_DEPS, ...options.deps }; @@ -172,12 +173,20 @@ export async function runAutoSyncOnce( }) ) { try { - const analysis = await deps.runAnalysis( - targetDir, - { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, - config.analyzeTimeoutMs, - options.signal, - ); + const analysis = await (options.onAnalysisCancellationRequested + ? deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + options.onAnalysisCancellationRequested, + ) + : deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + )); throwIfAborted(options.signal); stats = analysis.stats; analyzeStatus = 'success'; @@ -338,7 +347,7 @@ export async function runAutoSyncOnce( } function shortErrorMessage(err: unknown): string { - const message = (err as Error).message || String(err); + const message = err instanceof Error ? err.message : String(err); return message.replace(/\s+/g, ' ').slice(0, 240); } diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index 026ea512b..60883fd52 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -13,7 +13,7 @@ export interface AutoSyncStartHandle { stop(): Promise; } -export type WatchStatusState = 'running' | 'stopping' | 'stopped' | 'stale' | 'error'; +export type WatchStatusState = 'running' | 'cancelling' | 'stopping' | 'stopped' | 'stale' | 'error'; export type AutoSyncWatchStopResult = 'stopped' | 'not_running' | 'refused' | 'timeout'; export interface WatchStatusRecord { @@ -32,6 +32,15 @@ export interface WatchOwnerRecord { createdAt: string; } +interface WatchStopRequestRecord { + pid: number; + ownerId: string; + processStartTime: string; + requestedAt: string; +} + +const WATCH_STOP_POLL_MS = 250; + export interface AutoSyncWatchPaths { pidPath: string; mutexPath: string; @@ -43,7 +52,6 @@ export interface AutoSyncWatchControlDeps { isProcessAlive(pid: number): boolean; readProcessCommand(pid: number): string | undefined; readProcessStartTime(pid: number): string | undefined; - killProcess(pid: number, signal?: NodeJS.Signals): void; sleep(ms: number): Promise; } @@ -112,8 +120,26 @@ export async function startAutoSyncWatch( }); const runOnce = options.runOnce ?? runAutoSyncOnce; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; let activeRun: Promise | undefined; let activeAbortController: AbortController | undefined; + let stopping = false; + let statusWrite = Promise.resolve(); + const updateStatus = (state: WatchStatusState, message?: string) => { + const write = statusWrite.then(() => + writeWatchStatus(paths, { + state, + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + message, + updatedAt: new Date().toISOString(), + }), + ); + statusWrite = write.catch(() => {}); + return write; + }; const runSafely = () => { if (activeRun) { stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); @@ -122,7 +148,17 @@ export async function startAutoSyncWatch( const startedAt = new Date(); stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); const abortController = new AbortController(); - const run = runOnce(loaded.config, { signal: abortController.signal }) + const run = runOnce(loaded.config, { + signal: abortController.signal, + onAnalysisCancellationRequested: () => { + if (!stopping) { + void updateStatus( + 'cancelling', + 'Analysis cancellation requested; waiting for the worker to reach a safe shutdown point.', + ); + } + }, + }) .then((result) => { stderr.write( `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, @@ -131,50 +167,56 @@ export async function startAutoSyncWatch( .catch((err: unknown) => { stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); stderr.write('[auto-sync] Watch loop finished: failed.\n'); + }) + .finally(async () => { + if (activeRun === run) { + activeRun = undefined; + activeAbortController = undefined; + } + if (!stopping) await updateStatus('running'); }); activeRun = run; activeAbortController = abortController; - void run.finally(() => { - if (activeRun === run) { - activeRun = undefined; - activeAbortController = undefined; + }; + + let timer: ReturnType | undefined; + let controlTimer: ReturnType | undefined; + let stopPromise: Promise | undefined; + const stop = () => + (stopPromise ??= (async () => { + stopping = true; + if (timer) clearIntervalFn(timer); + if (controlTimer) clearIntervalFn(controlTimer); + activeAbortController?.abort(); + try { + await updateStatus('stopping'); + await activeRun?.catch(() => {}); + await updateStatus('stopped'); + } finally { + await cleanupWatchFiles(paths, ownerId, releaseLock); } - }); + })()); + const checkStopRequest = async () => { + const request = await readStopRequest(stopRequestPath(paths, ownerId)); + if ( + request?.pid === process.pid && + request.ownerId === ownerId && + request.processStartTime === processStartTime + ) { + void stop().catch((error: unknown) => { + stderr.write(`[auto-sync] Failed to stop watch: ${(error as Error).message}\n`); + }); + } }; runSafely(); - const intervalMs = loaded.config.syncIntervalMinutes * 60_000; - const setIntervalFn = options.setIntervalFn ?? setInterval; - const clearIntervalFn = options.clearIntervalFn ?? clearInterval; - const timer = setIntervalFn(runSafely, intervalMs); - if (options.keepAlive === false) timer.unref?.(); - let stopPromise: Promise | undefined; - return { - stop: () => - (stopPromise ??= (async () => { - clearIntervalFn(timer); - activeAbortController?.abort(); - try { - await writeWatchStatus(paths, { - state: 'stopping', - pid: process.pid, - ownerId, - configPath: loaded.config.configPath, - updatedAt: new Date().toISOString(), - }); - await activeRun?.catch(() => {}); - await writeWatchStatus(paths, { - state: 'stopped', - pid: process.pid, - ownerId, - configPath: loaded.config.configPath, - updatedAt: new Date().toISOString(), - }); - } finally { - await cleanupWatchFiles(paths, ownerId, releaseLock); - } - })()), - }; + controlTimer = setIntervalFn(() => void checkStopRequest(), WATCH_STOP_POLL_MS); + timer = setIntervalFn(runSafely, loaded.config.syncIntervalMinutes * 60_000); + if (options.keepAlive === false) { + controlTimer.unref?.(); + timer.unref?.(); + } + return { stop }; } catch (error) { await cleanupWatchFiles(paths, ownerId, releaseLock).catch(() => {}); throw error; @@ -276,8 +318,16 @@ export async function stopAutoSyncWatch( return 'refused'; } - deps.killProcess(pid, 'SIGTERM'); - stderr.write(`[auto-sync] Stop signal sent to watch pid ${pid}.\n`); + await writeAtomicText( + stopRequestPath(paths, owner.owner.ownerId), + `${JSON.stringify({ + pid, + ownerId: owner.owner.ownerId, + processStartTime: owner.owner.processStartTime, + requestedAt: new Date().toISOString(), + } satisfies WatchStopRequestRecord)}\n`, + ); + stderr.write(`[auto-sync] Stop requested for watch pid ${pid}.\n`); const stopped = await waitForProcessExit(pid, { deps, timeoutMs, pollMs }); if (!stopped) { stderr.write(`[auto-sync] Watch pid ${pid} did not exit within ${timeoutMs}ms.\n`); @@ -317,7 +367,10 @@ export async function readAutoSyncWatchStatus( } return { ...stored, - state: stored?.state === 'stopping' ? 'stopping' : 'running', + state: + stored?.state === 'cancelling' || stored?.state === 'stopping' + ? stored.state + : 'running', pid, ownerId: owner.owner.ownerId, updatedAt: new Date().toISOString(), @@ -426,6 +479,33 @@ async function readStatusFile(statusPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(filePath, 'utf-8')) as WatchStopRequestRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime && + typeof parsed.requestedAt === 'string' && + parsed.requestedAt + ) { + return parsed; + } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return undefined; + } + return undefined; +} + async function writeWatchStatus( paths: AutoSyncWatchPaths, record: WatchStatusRecord, @@ -452,6 +532,7 @@ async function cleanupWatchFiles( if ((await readOwnerFile(paths.ownerPath))?.ownerId === ownerId) { await removeIfExists(paths.ownerPath); } + await removeIfExists(stopRequestPath(paths, ownerId)); } } finally { await releaseLock(); @@ -505,11 +586,6 @@ function resolveWatchDeps(deps: Partial = {}): AutoSyn } }), readProcessStartTime: deps.readProcessStartTime ?? readProcessStartTime, - killProcess: - deps.killProcess ?? - ((pid, signal = 'SIGTERM') => { - process.kill(pid, signal); - }), sleep: deps.sleep ?? ((ms) => diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 81193598e..d573f5a5c 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -26,13 +26,20 @@ import type { AnalyzeOptions } from '../core/run-analyze.js'; import type { AnalyzeResultIpc } from './analyze-worker-ipc.js'; -/** Parent → child: the single command that starts an analysis run. */ +/** Parent → child: start one analysis run. */ export interface StartMessage { type: 'start'; repoPath: string; options: AnalyzeOptions; } +/** Parent → child: request safe cancellation at the next JS-visible checkpoint. */ +export interface CancelMessage { + type: 'cancel'; +} + +export type ParentMessage = StartMessage | CancelMessage; + export interface ProgressMessage { type: 'progress'; phase: string; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index ad4632eb0..762ebcc46 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -6,12 +6,13 @@ * * IPC Protocol: * Parent -> Child: { type: 'start', repoPath: string, options: AnalyzeOptions } + * Parent -> Child: { type: 'cancel' } * Child -> Parent: { type: 'progress', phase: string, percent: number, message: string } * Child -> Parent: { type: 'complete', result: AnalyzeResult } * Child -> Parent: { type: 'error', message: string } */ -import type { StartMessage, WorkerMessage } from './analyze-worker-protocol.js'; +import type { ParentMessage, WorkerMessage } from './analyze-worker-protocol.js'; import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js'; type BoundedCheckpointBeforeExit = typeof import('../core/lbug/shutdown-helpers.js').boundedCheckpointBeforeExit; @@ -62,19 +63,24 @@ process.on('unhandledRejection', (reason: unknown) => { } }); -// Handle cancellation / timeout shutdown (analyze-job.ts `cancelJob` sends -// SIGTERM). Bounded CHECKPOINT-then-exit shared with the CLI SIGINT path (#2264): -// skip the native close (the LadybugDB destructor can double-free after --pdg -// writes), but don't block behind the in-flight COPY's connection lock — so a -// single cancel can't abort or hang the worker. A CHECKPOINT failure is reported -// to the parent over IPC, not swallowed; the exit always fires. -process.on('SIGTERM', () => { - // Only report the cancellation if the analysis hasn't already reported a - // terminal outcome (#2264 P3) — otherwise this would flip an already-complete - // job to failed. The cleanup + exit below run regardless. +// IPC cancellation is the cross-platform control path. It only records the +// request while analysis is active; cleanup waits until the analysis promise has +// returned to JS. SIGTERM is retained only for local process shutdown. +let cancellationRequested = false; +let started = false; +function requestWorkerCancellation(source: string): void { + if (cancellationRequested) return; + cancellationRequested = true; if (claimTerminal()) { - send({ type: 'error', message: 'Analysis cancelled (worker received SIGTERM)' }); + send({ type: 'error', message: `Analysis cancelled (${source})` }); } + if (!started) { + // No analysis has started, so no native work needs a safe-point handshake. + process.exit(0); + } +} + +function exitAfterCancellation(): void { if (!boundedCheckpointBeforeExit) { process.exit(0); return; @@ -82,17 +88,21 @@ process.on('SIGTERM', () => { void boundedCheckpointBeforeExit({ exitCode: 0, onFlushError: (err: unknown) => { - const message = - err instanceof Error ? err.message : 'Worker checkpoint failed during SIGTERM'; + const message = err instanceof Error ? err.message : 'Worker checkpoint failed during cancellation'; send({ type: 'error', message }); }, }); -}); +} -// Listen for start command from parent — guarded against re-entry -let started = false; -process.on('message', async (msg: StartMessage) => { - if (msg.type !== 'start' || started) return; +process.on('SIGTERM', () => requestWorkerCancellation('worker received SIGTERM')); + +// Listen for parent commands — guarded against re-entry. +process.on('message', async (msg: ParentMessage) => { + if (msg.type === 'cancel') { + requestWorkerCancellation('parent requested cancellation'); + return; + } + if (started) return; started = true; try { @@ -112,6 +122,9 @@ process.on('message', async (msg: StartMessage) => { }, ); boundedCheckpointBeforeExit = prepared.loaded.shutdownHelpers.boundedCheckpointBeforeExit; + // A cancel can arrive while the dynamic imports are resolving. Do not begin + // a new analysis after that request; the finally block performs safe cleanup. + if (cancellationRequested) return; // The run → finalize → report contract lives in the side-effect-free // analyze-worker-core seam (unit-testable without this entry module's // process.on side effects). It reports exactly one terminal message and @@ -135,9 +148,11 @@ process.on('message', async (msg: StartMessage) => { }); } } finally { - // LadybugDB's native module prevents clean exit — force it (same reason the - // CLI uses process.exit(0)). In `finally` so the exit still fires even if the - // report above throws on a closed IPC channel (#2264 review P3). - setTimeout(() => process.exit(0), 500); + // A cancel must not end the process while runFullAnalysis may still be in + // native code. This continuation runs only after that promise has settled. + if (cancellationRequested) exitAfterCancellation(); + // Normal terminal outcomes still need the existing process exit because + // LadybugDB stays live. + else setTimeout(() => process.exit(0), 500); } }); diff --git a/gitnexus/src/storage/file-lock.ts b/gitnexus/src/storage/file-lock.ts index b509172b7..c47377cad 100644 --- a/gitnexus/src/storage/file-lock.ts +++ b/gitnexus/src/storage/file-lock.ts @@ -57,7 +57,7 @@ export async function acquireFileLock( await fs.link(pendingPath, resolvedPath); break; } catch (error) { - if (!isLockConflict(error)) throw error; + if (!(await isLockConflict(error, resolvedPath))) throw error; if ( await reclaimStaleLock( resolvedPath, @@ -158,7 +158,14 @@ async function readOwner(lockPath: string): Promise { return undefined; } -function isLockConflict(error: unknown): boolean { +async function isLockConflict(error: unknown, lockPath: string): Promise { const code = (error as NodeJS.ErrnoException).code; - return code === 'EEXIST' || code === 'EPERM'; + if (code === 'EEXIST') return true; + if (code !== 'EPERM') return false; + try { + await fs.access(lockPath); + return true; + } catch { + return false; + } } diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 9c302a013..2b9d304eb 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -638,12 +638,13 @@ export const readRegistry = async (): Promise => { * `withRegistryLock` degrades to unlocked on timeout, so the write cannot rely * on the lock to keep two writers off one staging path (#2888). */ -const writeRegistry = async (entries: RegistryEntry[]): Promise => { +const writeRegistry = async (entries: RegistryEntry[], attempts?: number): Promise => { const dir = getGlobalDir(); await fs.mkdir(dir, { recursive: true }); await writeFileAtomic( getGlobalRegistryPath(), JSON.stringify(sanitizeEntries(entries), null, 2), + attempts, ); }; @@ -1423,7 +1424,7 @@ export const listRegisteredRepos = async (opts?: { try { await withRegistryLock(async () => { const fresh = await readRegistry(); - await writeRegistry(fresh.filter((entry) => !pruned.has(entry.path))); + await writeRegistry(fresh.filter((entry) => !pruned.has(entry.path)), 1); }); } catch (err) { // Best-effort housekeeping: callers consume the returned view, and the diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts index d6a7b42d6..79769a976 100644 --- a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -8,14 +8,17 @@ vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; +function createChild() { + return Object.assign(new EventEmitter(), { + send: vi.fn(), + stdout: { resume: vi.fn() }, + stderr: { resume: vi.fn() }, + }); +} + describe('auto-sync analysis worker', () => { it('ignores progress and resolves from the terminal complete message', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - stdout: { resume: vi.fn() }, - stderr: { resume: vi.fn() }, - }); + const child = createChild(); const forkWorker = vi.fn(() => child as any); const run = createAutoSyncAnalysisRunner({ forkWorker }); @@ -24,7 +27,7 @@ describe('auto-sync analysis worker', () => { expect.any(String), expect.arrayContaining(['--max-old-space-size=512']), ); - child.emit('message', { type: 'progress', phase: 'parsing', progress: 20 }); + child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); child.emit('exit', 0, null); @@ -33,39 +36,46 @@ describe('auto-sync analysis worker', () => { expect(child.stderr.resume).toHaveBeenCalled(); }); - it('rejects immediately when the worker emits an error without exiting', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); + it('requests cancellation on a worker error but waits for its safe exit', async () => { + const child = createChild(); const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); - const result = run('/tmp/repo', { branch: 'main' }, 50); - child.emit('error', new Error('spawn failed')); + let settled = false; + void result.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); - await expect(result).rejects.toThrow('Auto-sync analyze worker error: spawn failed'); + child.emit('error', new Error('IPC disconnected')); + + await Promise.resolve(); + expect(settled).toBe(false); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + + child.emit('exit', 1, null); + await expect(result).rejects.toThrow('Auto-sync analyze worker error: IPC disconnected'); }); - it('preserves a worker error after progress messages', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); + it('preserves a worker terminal error', async () => { + const child = createChild(); const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); const result = run('/tmp/repo', { branch: 'main' }, 50); - child.emit('message', { type: 'progress', phase: 'parsing', progress: 20 }); + child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); child.emit('message', { type: 'error', message: 'parser crashed' }); child.emit('exit', 1, null); await expect(result).rejects.toThrow('parser crashed'); }); - it('waits for timed-out worker exit before releasing the scheduled run', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); + + it('requests cancellation after timeout, reports it, and waits for exit', async () => { + const child = createChild(); const timers: Array<() => void> = []; + const onCancellationRequested = vi.fn(); const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any), setTimeoutFn: vi.fn((callback: () => void) => { @@ -75,27 +85,29 @@ describe('auto-sync analysis worker', () => { clearTimeoutFn: vi.fn() as any, }); - const result = run('/tmp/repo', { branch: 'main' }, 50); - expect(child.send).toHaveBeenCalledWith({ - type: 'start', - repoPath: '/tmp/repo', - options: { branch: 'main' }, - }); + const result = run('/tmp/repo', { branch: 'main' }, 50, undefined, onCancellationRequested); + timers[0]!(); - timers[0](); - expect(child.kill).toHaveBeenCalledWith('SIGTERM'); - timers[1](); - expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + expect(onCancellationRequested).toHaveBeenCalledOnce(); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + let settled = false; + void result.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await Promise.resolve(); + expect(settled).toBe(false); - child.emit('exit', null, 'SIGKILL'); + child.emit('exit', 0, null); await expect(result).rejects.toThrow('Analysis timed out after 50ms'); }); - it('keeps the timeout outcome when complete arrives after termination begins', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); + it('keeps the timeout outcome when complete arrives after cancellation begins', async () => { + const child = createChild(); const timers: Array<() => void> = []; const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any), @@ -107,66 +119,44 @@ describe('auto-sync analysis worker', () => { }); const result = run('/tmp/repo', { branch: 'main' }, 50); - timers[0](); + timers[0]!(); child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); child.emit('exit', 0, null); await expect(result).rejects.toThrow('Analysis timed out after 50ms'); }); - it('clears the analysis deadline after complete before the worker exits', async () => { - vi.useFakeTimers(); - try { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); - const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + it('does not send cancellation after a terminal complete message', async () => { + const child = createChild(); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); - const result = run('/tmp/repo', { branch: 'main' }, 50); - child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); - await vi.advanceTimersByTimeAsync(50); + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); - expect(child.kill).not.toHaveBeenCalled(); - child.emit('exit', 0, null); - await expect(result).resolves.toEqual({ stats: { files: 3 } }); - } finally { - vi.useRealTimers(); - } + await expect(result).resolves.toEqual({ stats: { files: 3 } }); + expect(child.send).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(1); }); - it('keeps the cancellation outcome when complete arrives after abort', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); + it('uses the same cancellation request for an aborted watch run', async () => { + const child = createChild(); + const controller = new AbortController(); const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); - const controller = new AbortController(); const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); controller.abort(); - child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + child.emit('exit', 0, null); - - await expect(result).rejects.toThrow('Analysis cancelled'); - }); - - it('asks an active worker to stop gracefully when watch is stopped', async () => { - const child = Object.assign(new EventEmitter(), { - send: vi.fn(), - kill: vi.fn(), - }); - const run = createAutoSyncAnalysisRunner({ - forkWorker: vi.fn(() => child as any), - }); - const controller = new AbortController(); - - const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); - controller.abort(); - - expect(child.kill).toHaveBeenCalledWith('SIGTERM'); - expect(child.kill).not.toHaveBeenCalledWith('SIGKILL'); - child.emit('exit', null, 'SIGTERM'); await expect(result).rejects.toThrow('Analysis cancelled'); }); }); diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index b5a1e4d50..591038368 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -362,8 +362,9 @@ describe('auto-sync runner', () => { ); }); - it('passes the watch stop signal to the isolated analysis runner', async () => { + it('passes watch cancellation controls to the isolated analysis runner', async () => { const controller = new AbortController(); + const onAnalysisCancellationRequested = vi.fn(); const runAnalysis = vi.fn(async () => ({ stats: { files: 1 } }) as any); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), @@ -383,6 +384,7 @@ describe('auto-sync runner', () => { deps, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, signal: controller.signal, + onAnalysisCancellationRequested, }); expect(runAnalysis).toHaveBeenCalledWith( @@ -390,6 +392,7 @@ describe('auto-sync runner', () => { { branch: 'master', skipAgentsMd: true, skipSkills: true }, 1_800_000, controller.signal, + onAnalysisCancellationRequested, ); }); @@ -956,6 +959,40 @@ describe('auto-sync runner', () => { ); }); + it('records a null analysis failure without masking it with a TypeError', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => { + throw null; + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }), + ).resolves.toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + lastAnalyzeError: 'null', + }), + }), + ); + }); + it('retries analysis on a new commit after consecutive failures reached the threshold', async () => { const errorLogger = vi.fn(); const deps: Partial = withCloneRoot({ @@ -1366,7 +1403,6 @@ describe('auto-sync starter', () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); const stderr = { write: vi.fn() }; - const killProcess = vi.fn(); try { await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); await fs.mkdir(paths.mutexPath); @@ -1379,11 +1415,10 @@ describe('auto-sync starter', () => { stopAutoSyncWatch({ paths, stderr, - deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, + deps: { isProcessAlive: vi.fn(() => true) }, }), ).resolves.toBe('refused'); - expect(killProcess).not.toHaveBeenCalled(); expect(stderr.write).toHaveBeenCalledWith( '[auto-sync] Watch appears to be starting with pid 12345; pid file is not ready.\n', ); @@ -1410,11 +1445,7 @@ describe('auto-sync starter', () => { stopAutoSyncWatch({ paths, stderr: { write: vi.fn() }, - deps: { - isProcessAlive: vi.fn(() => false), - killProcess: vi.fn(), - sleep: vi.fn(async () => {}), - }, + deps: { isProcessAlive: vi.fn(() => false) }, }), ).resolves.toBe('refused'); @@ -1440,160 +1471,35 @@ describe('auto-sync starter', () => { } }); - it('reports status and signals the verified owner without deleting its files', async () => { + it('writes an owner-fenced stop request without deleting a live watch lease', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); - const killProcess = vi.fn(); - let alive = true; try { - await writeWatchOwner(paths, 12345); - - await expect( - readAutoSyncWatchStatus(paths, { - isProcessAlive: vi.fn(() => true), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - }), - ).resolves.toMatchObject({ state: 'running', pid: 12345 }); - await expect( - stopAutoSyncWatch({ - paths, - stderr: { write: vi.fn() }, - pollMs: 1, - deps: { - isProcessAlive: vi.fn(() => alive), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess: vi.fn((pid, signal) => { - killProcess(pid, signal); - alive = false; - }), - sleep: vi.fn(async () => {}), - }, - }), - ).resolves.toBe('stopped'); - - expect(killProcess).toHaveBeenCalledWith(12345, 'SIGTERM'); - await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); - await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); - await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); - expect(JSON.parse(await fs.readFile(paths.statusPath, 'utf-8'))).toMatchObject({ - state: 'running', - pid: 12345, - }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it('does not delete or overwrite successor ownership after the old owner exits', async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); - const paths = getAutoSyncWatchPaths(tempDir); - let oldOwnerAlive = true; - let handedOver = false; - try { - await writeWatchOwner(paths, 12345, 'old-owner'); + const ownerId = await writeWatchOwner(paths, 12345, 'verified-owner'); await expect( stopAutoSyncWatch({ paths, + timeoutMs: 0, stderr: { write: vi.fn() }, - pollMs: 1, - deps: { - isProcessAlive: vi.fn((pid) => (pid === 12345 ? oldOwnerAlive : true)), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess: vi.fn(), - sleep: vi.fn(async () => { - if (handedOver) return; - handedOver = true; - oldOwnerAlive = false; - await fs.writeFile(paths.pidPath, '54321\n'); - await fs.writeFile( - paths.ownerPath, - `${JSON.stringify({ pid: 54321, ownerId: 'successor', processStartTime: verifiedProcessStartTime, createdAt: '2026-08-04T00:00:00.000Z' })}\n`, - ); - await fs.writeFile( - paths.statusPath, - `${JSON.stringify({ state: 'running', pid: 54321, ownerId: 'successor', updatedAt: '2026-08-04T00:00:00.000Z' })}\n`, - ); - }), - }, - }), - ).resolves.toBe('stopped'); - - await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('54321\n'); - expect(JSON.parse(await fs.readFile(paths.ownerPath, 'utf-8'))).toMatchObject({ - pid: 54321, - ownerId: 'successor', - }); - expect(JSON.parse(await fs.readFile(paths.statusPath, 'utf-8'))).toMatchObject({ - state: 'running', - pid: 54321, - ownerId: 'successor', - }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it('refuses to signal when the process command is unavailable', async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); - const paths = getAutoSyncWatchPaths(tempDir); - const killProcess = vi.fn(); - try { - await writeWatchOwner(paths, 12345); - - await expect( - stopAutoSyncWatch({ - paths, - stderr: { write: vi.fn() }, - pollMs: 1, - deps: { - isProcessAlive: vi.fn(() => true), - readProcessCommand: vi.fn(() => undefined), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess, - sleep: vi.fn(async () => {}), - }, - }), - ).resolves.toBe('refused'); - expect(killProcess).not.toHaveBeenCalled(); - await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it('does not change owner status when stop times out', async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); - const paths = getAutoSyncWatchPaths(tempDir); - try { - await writeWatchOwner(paths, 12345); - - await expect( - stopAutoSyncWatch({ - paths, - stderr: { write: vi.fn() }, - timeoutMs: 2, - pollMs: 1, deps: { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => verifiedWatchCommand), readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess: vi.fn(), - sleep: vi.fn(async () => {}), }, }), ).resolves.toBe('timeout'); - await expect( - readAutoSyncWatchStatus(paths, { - isProcessAlive: vi.fn(() => true), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - }), - ).resolves.toMatchObject({ state: 'running', pid: 12345 }); + expect( + JSON.parse( + await fs.readFile(path.join(path.dirname(paths.pidPath), `watch.stop.${ownerId}.json`), 'utf-8'), + ), + ).toMatchObject({ + pid: 12345, + ownerId, + processStartTime: verifiedProcessStartTime, + requestedAt: expect.any(String), + }); await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); @@ -1602,85 +1508,9 @@ describe('auto-sync starter', () => { } }); - it('refuses to stop when pid status and lock ownership disagree', async () => { + it('refuses to request stop for a reused pid', async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); - const killProcess = vi.fn(); - try { - await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); - await fs.writeFile(paths.pidPath, '12345\n'); - await fs.writeFile( - paths.ownerPath, - `${JSON.stringify({ pid: 12345, ownerId: 'lock-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, - ); - await fs.writeFile( - paths.statusPath, - `${JSON.stringify({ state: 'running', pid: 12345, ownerId: 'other-owner', updatedAt: '2026-06-30T00:00:00.000Z' })}\n`, - ); - - await expect( - stopAutoSyncWatch({ - paths, - stderr: { write: vi.fn() }, - deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, - }), - ).resolves.toBe('refused'); - - expect(killProcess).not.toHaveBeenCalled(); - await expect( - readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), - ).resolves.toMatchObject({ - state: 'error', - pid: 12345, - message: expect.stringContaining('owner'), - }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it('refuses to signal a reused pid even when it is another GitNexus watch', async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); - const paths = getAutoSyncWatchPaths(tempDir); - const killProcess = vi.fn(); - try { - await writeWatchOwner(paths, 12345); - - await expect( - stopAutoSyncWatch({ - paths, - stderr: { write: vi.fn() }, - deps: { - isProcessAlive: vi.fn(() => true), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => 'Tue Aug 4 13:00:00 2026'), - killProcess, - sleep: vi.fn(async () => {}), - }, - }), - ).resolves.toBe('refused'); - - expect(killProcess).not.toHaveBeenCalled(); - await expect( - readAutoSyncWatchStatus(paths, { - isProcessAlive: vi.fn(() => true), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => 'Tue Aug 4 13:00:00 2026'), - }), - ).resolves.toMatchObject({ - state: 'error', - pid: 12345, - message: expect.stringContaining('different process'), - }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it('refuses to signal a reused pid whose command is not GitNexus watch', async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); - const paths = getAutoSyncWatchPaths(tempDir); - const killProcess = vi.fn(); try { await writeWatchOwner(paths, 12345); @@ -1692,13 +1522,10 @@ describe('auto-sync starter', () => { isProcessAlive: vi.fn(() => true), readProcessCommand: vi.fn(() => 'node unrelated-service.js'), readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess, - sleep: vi.fn(async () => {}), }, }), ).resolves.toBe('refused'); - expect(killProcess).not.toHaveBeenCalled(); await expect( readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true), @@ -1710,83 +1537,149 @@ describe('auto-sync starter', () => { pid: 12345, message: expect.stringContaining('not a GitNexus watch process'), }); + await expect(fs.readdir(path.dirname(paths.pidPath))).resolves.not.toContainEqual( + expect.stringMatching(/^watch\.stop\./), + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); - it('restart starts only after the owner releases its mutex', async () => { + it('stops a watch only when its own owner-fenced request is polled', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); const paths = getAutoSyncWatchPaths(tempDir); + const callbacks: Array<() => void> = []; const timer = { unref: vi.fn() }; - const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; - const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; - const runOnce = vi.fn(async () => ({ - synced: 0, - analyzed: 0, - skippedAnalysis: 0, - failed: 0, - })); - let alive = true; - let ownerStop: Promise | undefined; try { process.env.GITNEXUS_HOME = tempDir; await fs.writeFile( path.join(tempDir, 'watch_config.yml'), - [ - 'sync_interval_minutes: 5', - 'projects:', - ' - local_path: /tmp/repos', - ' branch: master', - ' remote_urls:', - ' - git@github.com:team/repo.git', - ].join('\n'), + ['sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', ' branch: master', ' remote_urls:', ' - git@github.com:team/repo.git'].join('\n'), ); - const ownerHandle = await startAutoSyncWatch({ + const handle = await startAutoSyncWatch({ paths, - setIntervalFn, - clearIntervalFn, - runOnce, keepAlive: false, + setIntervalFn: vi.fn((callback: () => void) => { + callbacks.push(callback); + return timer; + }) as unknown as typeof setInterval, + clearIntervalFn: vi.fn() as unknown as typeof clearInterval, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, }); - expect(ownerHandle).not.toBeNull(); + expect(handle).not.toBeNull(); + const owner = JSON.parse(await fs.readFile(paths.ownerPath, 'utf-8')); + await fs.writeFile( + path.join(path.dirname(paths.pidPath), `watch.stop.${owner.ownerId}.json`), + `${JSON.stringify({ + pid: process.pid, + ownerId: owner.ownerId, + processStartTime: verifiedProcessStartTime, + requestedAt: new Date().toISOString(), + })}\n`, + ); - await expect( - stopAutoSyncWatch({ - paths, - timeoutMs: 10, - pollMs: 1, - stderr: { write: vi.fn() }, - deps: { - isProcessAlive: vi.fn(() => alive), - readProcessCommand: vi.fn(() => verifiedWatchCommand), - readProcessStartTime: vi.fn(() => verifiedProcessStartTime), - killProcess: vi.fn(() => { - ownerStop = ownerHandle!.stop().then(() => { - alive = false; - }); + callbacks[0]!(); + await vi.waitFor(async () => expect(fs.access(paths.pidPath)).rejects.toThrow()); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports cancelling until a timed-out analysis run settles', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + let releaseRun!: () => void; + let requestCancellation!: () => void; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + ['sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', ' branch: master', ' remote_urls:', ' - git@github.com:team/repo.git'].join('\n'), + ); + const handle = await startAutoSyncWatch({ + paths, + keepAlive: false, + runOnce: vi.fn( + (_config, options) => + new Promise((resolve) => { + requestCancellation = options.onAnalysisCancellationRequested; + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); }), - sleep: vi.fn(async () => { - await ownerStop; + ), + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + requestCancellation(); + await vi.waitFor(async () => { + await expect(readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + })).resolves.toMatchObject({ state: 'cancelling' }); + }); + + releaseRun(); + await vi.waitFor(async () => { + await expect(readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + })).resolves.toMatchObject({ state: 'running' }); + }); + await handle?.stop(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('keeps watch ownership while stop waits for an active run to settle', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + let releaseRun!: () => void; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + ['sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', ' branch: master', ' remote_urls:', ' - git@github.com:team/repo.git'].join('\n'), + ); + const handle = await startAutoSyncWatch({ + paths, + keepAlive: false, + runOnce: vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); }), - }, - }), - ).resolves.toBe('stopped'); + ), + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + const stopping = handle!.stop(); + + await vi.waitFor(async () => { + await expect(readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + })).resolves.toMatchObject({ state: 'stopping' }); + }); + await expect(fs.access(paths.pidPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + + releaseRun(); + await stopping; await expect(fs.access(paths.pidPath)).rejects.toThrow(); await expect(fs.access(paths.ownerPath)).rejects.toThrow(); await expect(fs.access(paths.mutexPath)).rejects.toThrow(); - - const successorHandle = await startAutoSyncWatch({ - paths, - setIntervalFn, - clearIntervalFn, - runOnce, - keepAlive: false, - }); - expect(successorHandle).not.toBeNull(); - await successorHandle?.stop(); } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 668671761..f000e5e51 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -458,6 +458,10 @@ describe('auto-sync', () => { expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo.git#main')).toThrow( 'must not include query strings or fragments', ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/')).toThrow('path must include'); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner//repo')).toThrow( + 'path must include', + ); }); it('parses repo git timeout durations', () => { diff --git a/gitnexus/test/unit/file-lock.test.ts b/gitnexus/test/unit/file-lock.test.ts index 8c2a41abd..aae3cbf43 100644 --- a/gitnexus/test/unit/file-lock.test.ts +++ b/gitnexus/test/unit/file-lock.test.ts @@ -2,7 +2,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { acquireFileLock, FileLockBusyError } from '../../src/storage/file-lock.js'; @@ -28,6 +28,18 @@ describe('file lock', () => { await release(); }); + it('propagates hard-link EPERM when no lock exists', async () => { + const lockPath = await tempLockPath(); + const error = Object.assign(new Error('hard links unavailable'), { code: 'EPERM' }); + const link = vi.spyOn(fs, 'link').mockRejectedValueOnce(error); + + try { + await expect(acquireFileLock(lockPath)).rejects.toBe(error); + } finally { + link.mockRestore(); + } + }); + it('releases idempotently', async () => { const lockPath = await tempLockPath(); const release = await acquireFileLock(lockPath); diff --git a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts index a30005ed3..6a56c6bb0 100644 --- a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts @@ -171,8 +171,10 @@ describe('writeRegistry — private tmp path per transaction (#2888)', () => { it('keeps serving a validating read when the prune write fails', async () => { await registerRepo(tmpRepoA.dbPath, meta, { name: 'gone' }); fsCtx.renameMock.mockClear(); + // EBUSY is normally retryable, but prune persistence is best-effort and + // must not hold the registry lock through retry backoff. fsCtx.renameMock.mockImplementationOnce(() => - Promise.reject(Object.assign(new Error('mock read-only home'), { code: 'EROFS' })), + Promise.reject(Object.assign(new Error('mock busy registry'), { code: 'EBUSY' })), ); const cap = _captureLogger();