From bf1f38b9aa7d2df5a6362c591c4038a66fa33db7 Mon Sep 17 00:00:00 2001 From: weiyf Date: Tue, 30 Jun 2026 20:40:00 +0800 Subject: [PATCH] adds an opt-in auto sync and analysis loop for GitNexus --- .gitignore | 3 + gitnexus/src/cli/index.ts | 6 + gitnexus/src/core/auto-sync/config.ts | 163 +++++++ gitnexus/src/core/auto-sync/index.ts | 40 ++ gitnexus/src/core/auto-sync/path-security.ts | 173 ++++++++ gitnexus/src/core/auto-sync/repo.ts | 5 + gitnexus/src/core/auto-sync/runner.ts | 201 +++++++++ gitnexus/src/core/auto-sync/starter.ts | 51 +++ gitnexus/src/core/auto-sync/state.ts | 61 +++ gitnexus/src/server/git-clone.ts | 115 ++++- gitnexus/test/unit/auto-sync-runner.test.ts | 421 +++++++++++++++++++ gitnexus/test/unit/auto-sync.test.ts | 250 +++++++++++ gitnexus/test/unit/git-clone.test.ts | 194 ++++++++- 13 files changed, 1656 insertions(+), 27 deletions(-) create mode 100644 gitnexus/src/core/auto-sync/config.ts create mode 100644 gitnexus/src/core/auto-sync/index.ts create mode 100644 gitnexus/src/core/auto-sync/path-security.ts create mode 100644 gitnexus/src/core/auto-sync/repo.ts create mode 100644 gitnexus/src/core/auto-sync/runner.ts create mode 100644 gitnexus/src/core/auto-sync/starter.ts create mode 100644 gitnexus/src/core/auto-sync/state.ts create mode 100644 gitnexus/test/unit/auto-sync-runner.test.ts create mode 100644 gitnexus/test/unit/auto-sync.test.ts diff --git a/.gitignore b/.gitignore index 11f2743c7..bf7f113a8 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ npm-debug.log* # Testing coverage/ +.tmp-test/ +gitnexus/.tmp-test/ # Misc *.local @@ -110,3 +112,4 @@ local_docs/ .agents/ .context/ gitnexus/web/ +/log/ diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 72e4da365..4426eec9d 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -15,6 +15,12 @@ const _require = createRequire(import.meta.url); const pkg = _require('../../package.json'); const program = new Command(); +if (process.env.AUTO_UPDATE_AND_ANALYZE_FLAG?.trim() === '1') { + void import('../core/auto-sync/index.js').then(({ maybeStartAutoSyncFromEnv }) => + maybeStartAutoSyncFromEnv(), + ); +} + function collectCodingAgents(value: string, previous: string[] | undefined): string[] { return [...(previous ?? []), ...value.split(',')]; } diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts new file mode 100644 index 000000000..9c0a67e99 --- /dev/null +++ b/gitnexus/src/core/auto-sync/config.ts @@ -0,0 +1,163 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { normalizeConfiguredCloneRoot } from './path-security.js'; + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +export const AUTO_SYNC_FLAG = 'AUTO_UPDATE_AND_ANALYZE_FLAG'; +export const AUTO_SYNC_CONFIG_FILE = 'sync_config.yml'; +const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +const MIN_SYNC_INTERVAL_MINUTES = 5; + +export interface AutoSyncProjectConfig { + localPath: string; + gitnexusGroup?: string; + branches: string[]; + remoteUrls: string[]; +} + +export interface AutoSyncConfig { + configPath: string; + syncIntervalMinutes: number; + projects: AutoSyncProjectConfig[]; +} + +export type AutoSyncFlagDecision = + | { enabled: true } + | { enabled: false; reason: 'unset' | 'disabled' | 'invalid'; message?: string }; + +export type AutoSyncConfigLoadResult = + | { ok: true; config: AutoSyncConfig } + | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; + +export function parseAutoSyncFlag(raw = process.env[AUTO_SYNC_FLAG]): AutoSyncFlagDecision { + if (raw === undefined || raw.trim() === '') return { enabled: false, reason: 'unset' }; + const trimmed = raw.trim(); + if (trimmed === '0') return { enabled: false, reason: 'disabled' }; + if (trimmed === '1') return { enabled: true }; + return { + enabled: false, + reason: 'invalid', + message: `[auto-sync] ${AUTO_SYNC_FLAG} must be 0 or 1; got "${trimmed}". Auto sync is disabled.`, + }; +} + +export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); +} + +export function parseBranchCandidates(branchValue: unknown): string[] { + const rawItems = Array.isArray(branchValue) + ? branchValue.flatMap((item) => String(item).split(',')) + : String(branchValue ?? '').split(','); + const branches: string[] = []; + const seen = new Set(); + for (const item of rawItems) { + const branch = item.trim(); + if (!branch || seen.has(branch)) continue; + seen.add(branch); + branches.push(branch); + } + return branches; +} + +export async function loadAutoSyncConfig( + configPath = getAutoSyncConfigPath(), +): Promise { + let content: string; + try { + content = await fs.readFile(configPath, 'utf-8'); + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${configPath}. Auto sync is skipped.`, + }; + } + return { + ok: false, + reason: 'unreadable', + message: `[auto-sync] Unable to read config file: ${configPath}. Auto sync is skipped.`, + }; + } + + try { + return { ok: true, config: parseAutoSyncConfig(content, configPath) }; + } catch (err: unknown) { + return { + ok: false, + reason: 'invalid', + message: `[auto-sync] Invalid sync_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + }; + } +} + +export function parseAutoSyncConfig(content: string, configPath: string): AutoSyncConfig { + const raw = yaml.load(content, { schema: yaml.JSON_SCHEMA }) as Record; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('expected a YAML object'); + } + + const errors: string[] = []; + const interval = Number(raw.sync_interval_minutes); + if (!Number.isInteger(interval) || interval <= 0) { + errors.push('sync_interval_minutes must be a positive integer'); + } else if (interval < MIN_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); + } + + const rawProjects = raw.projects; + if (!Array.isArray(rawProjects) || rawProjects.length === 0) { + errors.push('projects must contain at least one project'); + } + + const projects: AutoSyncProjectConfig[] = []; + if (Array.isArray(rawProjects)) { + rawProjects.forEach((projectValue, index) => { + const project = projectValue as Record; + if (!project || typeof project !== 'object' || Array.isArray(project)) { + errors.push(`projects[${index}] must be an object`); + return; + } + + const localPath = typeof project.local_path === 'string' ? project.local_path.trim() : ''; + if (!localPath) { + errors.push(`projects[${index}].local_path is required`); + } else { + try { + normalizeConfiguredCloneRoot(localPath); + } catch (err: unknown) { + errors.push(`projects[${index}].local_path ${(err as Error).message}`); + } + } + + const remoteUrls = Array.isArray(project.remote_urls) + ? project.remote_urls.map((url) => String(url).trim()).filter(Boolean) + : []; + if (remoteUrls.length === 0) { + errors.push(`projects[${index}].remote_urls must contain at least one URL`); + } + + const branches = parseBranchCandidates(project.branch); + if (branches.length === 0) errors.push(`projects[${index}].branch is required`); + + const gitnexusGroup = + typeof project.gitnexus_group === 'string' ? project.gitnexus_group.trim() : undefined; + if (gitnexusGroup && !GROUP_NAME_PATTERN.test(gitnexusGroup)) { + errors.push(`projects[${index}].gitnexus_group is invalid`); + } + + if (localPath && remoteUrls.length > 0 && branches.length > 0) { + projects.push({ localPath, gitnexusGroup, branches, remoteUrls }); + } + }); + } + + if (errors.length > 0) throw new Error(errors.join('; ')); + return { configPath, syncIntervalMinutes: interval, projects }; +} diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts new file mode 100644 index 000000000..de1c0b4f7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/index.ts @@ -0,0 +1,40 @@ +export { + AUTO_SYNC_CONFIG_FILE, + AUTO_SYNC_FLAG, + getAutoSyncConfigPath, + loadAutoSyncConfig, + parseAutoSyncConfig, + parseAutoSyncFlag, + parseBranchCandidates, + type AutoSyncConfig, + type AutoSyncConfigLoadResult, + type AutoSyncFlagDecision, + type AutoSyncProjectConfig, +} from './config.js'; +export { + buildStateKey, + getAutoSyncStatePath, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitState, + type AutoSyncCommitStateEntry, +} from './state.js'; +export { extractRepoNameFromRemoteUrl } from './repo.js'; +export { + normalizeConfiguredCloneRoot, + quarantineAutoSyncPartial, + resolveConfiguredCloneRoot, + type AutoSyncCloneRoot, +} from './path-security.js'; +export { + addRepoToGroup, + getConfiguredRepoPath, + runAutoSyncOnce, + syncGroupByName, + type AutoSyncLogger, + type AutoSyncRunDeps, + type AutoSyncRunResult, +} from './runner.js'; +export { maybeStartAutoSyncFromEnv, type AutoSyncStartHandle } from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts new file mode 100644 index 000000000..10f5d6ab7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -0,0 +1,173 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +const DANGEROUS_ROOTS = new Set( + [ + '/', + os.homedir(), + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ].map((entry) => path.resolve(entry)), +); + +const DANGEROUS_PARENT_ROOTS = new Set( + [ + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ].map((entry) => path.resolve(entry)), +); + +const QUARANTINE_RETENTION_DAYS = 14; + +export interface AutoSyncCloneRoot { + root: string; + quarantineRoot: string; + quarantineRetentionDays: number; +} + +export async function resolveConfiguredCloneRoot(localPath: string): Promise { + const root = normalizeConfiguredCloneRoot(localPath); + assertNotDangerousRoot(root); + await assertNoSymlinkPath(root); + await assertDirectoryOwnerAndPermissions(root); + const realRoot = await fs.realpath(root); + assertContainedOrSame(root, realRoot, 'Configured clone root realpath escaped its normalized path'); + assertNotDangerousRoot(realRoot); + assertNotGitNexusInternalRoot(realRoot); + + return { + root: realRoot, + quarantineRoot: path.join(getGlobalDir(), 'quarantine'), + quarantineRetentionDays: QUARANTINE_RETENTION_DAYS, + }; +} + +export function normalizeConfiguredCloneRoot(localPath: string): string { + const value = localPath.trim(); + if (!value) throw new Error('local_path is required'); + if (!path.isAbsolute(value)) throw new Error('local_path must be an absolute path'); + if (value.split(path.sep).includes('..')) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + const resolved = path.resolve(value); + if (resolved !== path.normalize(value)) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + return resolved; +} + +export async function quarantineAutoSyncPartial(targetDir: string, quarantineRoot: string): Promise { + await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); + const base = path.basename(targetDir); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const destination = path.join(quarantineRoot, `auto-sync-${stamp}-${process.pid}-${base}`); + await fs.rename(targetDir, destination); + await fs.writeFile( + `${destination}.README.txt`, + [ + 'GitNexus auto-sync isolated a partial or unsafe clone result.', + `Created at: ${new Date().toISOString()}`, + `Original path: ${targetDir}`, + `Retention: keep for ${QUARANTINE_RETENTION_DAYS} days unless an operator reviews and removes it earlier.`, + 'Cleanup: verify the original path and remote before manual deletion.', + '', + ].join('\n'), + 'utf-8', + ); + return destination; +} + +function assertNotDangerousRoot(root: string): void { + if (DANGEROUS_ROOTS.has(root)) throw new Error(`Refusing unsafe auto-sync clone root: ${root}`); + for (const dangerousRoot of DANGEROUS_PARENT_ROOTS) { + const rel = path.relative(dangerousRoot, root); + if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) { + throw new Error(`Refusing unsafe auto-sync clone root under ${dangerousRoot}: ${root}`); + } + } + if (path.parse(root).root === root) throw new Error(`Refusing filesystem root as clone root: ${root}`); +} + +function assertNotGitNexusInternalRoot(root: string): void { + const gitnexusDir = path.resolve(getGlobalDir()); + const blocked = [ + gitnexusDir, + path.join(gitnexusDir, 'groups'), + path.join(gitnexusDir, 'indexes'), + path.join(gitnexusDir, 'quarantine'), + ]; + for (const blockedRoot of blocked) { + const rel = path.relative(blockedRoot, root); + if (!rel || (!rel.startsWith('..') && !path.isAbsolute(rel))) { + throw new Error(`Refusing GitNexus internal directory as auto-sync clone root: ${root}`); + } + } +} + +async function assertNoSymlinkPath(root: string): Promise { + const parsed = path.parse(root); + let current = parsed.root; + const parts = root.slice(parsed.root.length).split(path.sep).filter(Boolean); + for (const part of parts) { + current = path.join(current, part); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); + } +} + +export async function assertDirectoryOwnerAndPermissions(root: string): Promise { + const stat = await fs.stat(root); + if (!stat.isDirectory()) throw new Error(`auto-sync clone root is not a directory: ${root}`); + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`auto-sync clone root is owned by uid ${stat.uid}, not current process uid`); + } + const mode = stat.mode & 0o777; + const worldWritable = (mode & 0o002) !== 0; + const sticky = (stat.mode & 0o1000) !== 0; + if (worldWritable && !sticky) { + throw new Error(`Refusing world-writable auto-sync clone root without sticky bit: ${root}`); + } +} + +function assertContainedOrSame(root: string, child: string, message: string): void { + const rel = path.relative(root, child); + if (rel.startsWith('..') || path.isAbsolute(rel)) throw new Error(message); +} diff --git a/gitnexus/src/core/auto-sync/repo.ts b/gitnexus/src/core/auto-sync/repo.ts new file mode 100644 index 000000000..76dde2e49 --- /dev/null +++ b/gitnexus/src/core/auto-sync/repo.ts @@ -0,0 +1,5 @@ +import { extractRepoName } from '../../server/git-clone.js'; + +export function extractRepoNameFromRemoteUrl(remoteUrl: string): string { + return extractRepoName(remoteUrl); +} diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts new file mode 100644 index 000000000..c5e2232e4 --- /dev/null +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -0,0 +1,201 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import yaml from 'js-yaml'; +import { loadGroupConfig } from '../group/config-parser.js'; +import { getDefaultGitnexusDir, getGroupDir } from '../group/storage.js'; +import { syncGroup } from '../group/sync.js'; +import { runFullAnalysis } from '../run-analyze.js'; +import { getCurrentBranch, getCurrentCommit } from '../../storage/git.js'; +import { registerRepo, type RepoMeta } from '../../storage/repo-manager.js'; +import { extractRepoNameFromRemoteUrl } from './repo.js'; +import { cloneOrPull } from '../../server/git-clone.js'; +import { resolveConfiguredCloneRoot } from './path-security.js'; +import { + buildStateKey, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + type AutoSyncAnalyzeStatus, +} from './state.js'; +import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js'; + +export interface AutoSyncLogger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export interface AutoSyncRunDeps { + cloneOrPull: typeof cloneOrPull; + getCurrentBranch: typeof getCurrentBranch; + getCurrentCommit: typeof getCurrentCommit; + runFullAnalysis: typeof runFullAnalysis; + registerRepo: typeof registerRepo; + loadState: typeof loadAutoSyncState; + saveState: typeof saveAutoSyncState; + addRepoToGroup: typeof addRepoToGroup; + syncGroupByName: typeof syncGroupByName; + resolveCloneRoot: typeof resolveConfiguredCloneRoot; +} + +export interface AutoSyncRunResult { + synced: number; + analyzed: number; + skippedAnalysis: number; + failed: number; +} + +const DEFAULT_LOGGER: AutoSyncLogger = { + info: (message) => process.stderr.write(`${message}\n`), + warn: (message) => process.stderr.write(`${message}\n`), + error: (message) => process.stderr.write(`${message}\n`), +}; + +const DEFAULT_DEPS: AutoSyncRunDeps = { + cloneOrPull, + getCurrentBranch, + getCurrentCommit, + runFullAnalysis, + registerRepo, + loadState: loadAutoSyncState, + saveState: saveAutoSyncState, + addRepoToGroup, + syncGroupByName, + resolveCloneRoot: resolveConfiguredCloneRoot, +}; + +export async function runAutoSyncOnce( + config: AutoSyncConfig, + options: { deps?: Partial; logger?: AutoSyncLogger; now?: () => Date } = {}, +): Promise { + const deps = { ...DEFAULT_DEPS, ...options.deps }; + const logger = options.logger ?? DEFAULT_LOGGER; + const now = options.now ?? (() => new Date()); + const state = await deps.loadState(); + const groupsToSync = new Set(); + const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + + for (const project of config.projects) { + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const cloneRoot = await deps.resolveCloneRoot(project.localPath); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); + await deps.cloneOrPull(remoteUrl, targetDir, undefined, { + allowedCloneRoot: cloneRoot.root, + expectedRepoName: repoName, + quarantineRoot: cloneRoot.quarantineRoot, + }); + result.synced += 1; + + const currentBranch = deps.getCurrentBranch(targetDir); + if (!currentBranch || !project.branches.includes(currentBranch)) { + result.skippedAnalysis += 1; + logger.warn( + `[auto-sync] Skip analysis for ${targetDir}; current branch ${currentBranch ?? ''} is not in configured branches: ${project.branches.join(', ')}.`, + ); + continue; + } + + const branch = currentBranch; + const currentCommit = deps.getCurrentCommit(targetDir); + const stateKey = buildStateKey(targetDir, branch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + + if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runFullAnalysis( + targetDir, + { branch, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + const meta: RepoMeta = { + repoPath: targetDir, + lastCommit: currentCommit, + indexedAt: now().toISOString(), + stats: analysis.stats, + branch, + }; + await deps.registerRepo(targetDir, meta, { name: repoName, allowDuplicateName: true }); + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + result.analyzed += 1; + } catch (err: unknown) { + analyzeStatus = 'failed'; + result.failed += 1; + logger.error(`[auto-sync] Analysis failed for ${targetDir}: ${(err as Error).message}`); + } + } else { + result.skippedAnalysis += 1; + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + + state[stateKey] = { + codeCommitId: currentCommit, + analyzedCommitId, + lastAnalyzeStatus: analyzeStatus, + lastSyncTime: now().toISOString(), + }; + + if (project.gitnexusGroup) { + const added = await deps.addRepoToGroup(project, repoName); + if (added) groupsToSync.add(project.gitnexusGroup); + } + } catch (err: unknown) { + result.failed += 1; + logger.error(`[auto-sync] Repository sync failed for ${remoteUrl}: ${(err as Error).message}`); + } + } + } + + await deps.saveState(state); + for (const groupName of groupsToSync) { + try { + await deps.syncGroupByName(groupName); + } catch (err: unknown) { + result.failed += 1; + logger.error(`[auto-sync] Group sync failed for ${groupName}: ${(err as Error).message}`); + } + } + return result; +} + +export function getConfiguredRepoPath( + project: Pick, + repoName: string, +): string { + return path.resolve(project.localPath, repoName); +} + +export async function addRepoToGroup( + project: Pick, + repoName: string, +): Promise { + if (!project.gitnexusGroup) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.gitnexusGroup); + const config = await loadGroupConfig(groupDir); + if (Object.values(config.repos).includes(repoName)) return false; + config.repos[repoName] = repoName; + await writeGroupConfigAtomic(path.join(groupDir, 'group.yaml'), config); + return true; +} + +export async function syncGroupByName(groupName: string): Promise { + const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName); + const config = await loadGroupConfig(groupDir); + await syncGroup(config, { groupDir, allowStale: true }); +} + +async function writeGroupConfigAtomic(filePath: string, config: unknown): Promise { + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); + await fs.rename(tmpPath, filePath); +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts new file mode 100644 index 000000000..579b18ab5 --- /dev/null +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -0,0 +1,51 @@ +import { loadAutoSyncConfig, parseAutoSyncFlag } from './config.js'; +import { runAutoSyncOnce } from './runner.js'; + +export interface AutoSyncStartHandle { + stop(): void; +} + +export async function maybeStartAutoSyncFromEnv(options: { + setIntervalFn?: typeof setInterval; + clearIntervalFn?: typeof clearInterval; + runOnce?: typeof runAutoSyncOnce; + stderr?: Pick; +} = {}): Promise { + const stderr = options.stderr ?? process.stderr; + const flag = parseAutoSyncFlag(); + if (flag.enabled === false) { + if (flag.message) stderr.write(`${flag.message}\n`); + return null; + } + + const loaded = await loadAutoSyncConfig(); + if (loaded.ok === false) { + stderr.write(`${loaded.message}\n`); + return null; + } + + const runOnce = options.runOnce ?? runAutoSyncOnce; + let running = false; + const runSafely = () => { + if (running) { + stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); + return; + } + running = true; + void runOnce(loaded.config) + .catch((err: unknown) => { + stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + }) + .finally(() => { + running = false; + }); + }; + + runSafely(); + const intervalMs = loaded.config.syncIntervalMinutes * 60_000; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; + const timer = setIntervalFn(runSafely, intervalMs); + timer.unref?.(); + return { stop: () => clearIntervalFn(timer) }; +} diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts new file mode 100644 index 000000000..f14d9aa5f --- /dev/null +++ b/gitnexus/src/core/auto-sync/state.ts @@ -0,0 +1,61 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped'; + +export interface AutoSyncCommitStateEntry { + codeCommitId: string; + analyzedCommitId?: string; + lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + lastSyncTime: string; +} + +export type AutoSyncCommitState = Record; + +export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'auto-sync-state.json'); +} + +export function buildStateKey(repoPath: string, branch: string): string { + return `${path.resolve(repoPath)}|${branch}`; +} + +export function shouldAnalyzeCommit(input: { + currentCommit: string; + previousAnalyzedCommit?: string; + previousStatus?: AutoSyncAnalyzeStatus; +}): boolean { + if (!input.currentCommit) return false; + if (input.previousStatus === 'failed') return true; + return input.currentCommit !== input.previousAnalyzedCommit; +} + +export async function loadAutoSyncState( + statePath = getAutoSyncStatePath(), +): Promise { + try { + const raw = await fs.readFile(statePath, 'utf-8'); + const parsed = JSON.parse(raw); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as AutoSyncCommitState) + : {}; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + process.stderr.write( + `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + } + return {}; + } +} + +export async function saveAutoSyncState( + state: AutoSyncCommitState, + statePath = getAutoSyncStatePath(), +): Promise { + await fs.mkdir(path.dirname(statePath), { recursive: true }); + const tmpPath = `${statePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, statePath); +} diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 477f47a6e..588f36ea3 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -10,8 +10,11 @@ import path from 'path'; import fs from 'fs/promises'; import { isIP } from 'net'; import { logger } from '../core/logger.js'; -import { parseRepoNameFromUrl } from '../storage/git.js'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { + assertDirectoryOwnerAndPermissions, + quarantineAutoSyncPartial, +} from '../core/auto-sync/path-security.js'; /** * Root directory for all cloned repositories. Targets must resolve inside this. @@ -39,12 +42,16 @@ export const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; * clone root via path traversal. */ export function extractRepoName(url: string): string { - const name = parseRepoNameFromUrl(url); + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const name = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; if ( !name || name === '.' || name === '..' || name === 'unknown' || + name.startsWith('-') || !REPO_NAME_PATTERN.test(name) ) { throw new Error('Could not extract a valid repository name from URL'); @@ -235,6 +242,14 @@ export interface CloneProgress { message: string; } +export interface CloneOrPullOptions { + token?: string; + allowedCloneRoot?: string; + expectedRepoName?: string; + quarantineRoot?: string; + runGitForTest?: typeof runGit; +} + /** * Build the `git clone` argument list for a given URL and target directory. * @@ -443,36 +458,54 @@ export async function cloneOrPull( url: string, targetDir: string, onProgress?: (progress: CloneProgress) => void, - options?: { token?: string }, + options?: CloneOrPullOptions, ): Promise { // Containment barrier — inline with the canonical path.relative idiom so // CodeQL recognizes the sanitizer at every following filesystem and // subprocess sink. The same `safeTarget` is used for every downstream // path operation — no reassignment that the analyzer could lose track of. // - // Limitation: this is a lexical containment check, not a realpath check. - // If an attacker can place a symlink under CLONE_ROOT pointing outside it, - // the lexical check passes but the clone lands at the symlink target. That - // requires pre-existing local write access to CLONE_ROOT, so the threat - // model considers it out of scope; CodeQL js/path-injection accepts the - // lexical form. Tracked as a follow-up if defense-in-depth is needed. + // The lexical check runs before filesystem creation; realpath and symlink + // checks below run before pull/clone and again after clone completes. + const cloneRoot = path.resolve(options?.allowedCloneRoot ?? CLONE_ROOT); + const expectedRepoName = options?.expectedRepoName; + if (expectedRepoName !== undefined && expectedRepoName !== extractRepoName(url)) { + throw new Error(`Clone target repo name ${expectedRepoName} does not match requested URL`); + } + const safeTarget = path.resolve(targetDir); - const rel = path.relative(CLONE_ROOT, safeTarget); + if (expectedRepoName !== undefined && path.basename(safeTarget) !== expectedRepoName) { + throw new Error(`Clone target basename must match repository name ${expectedRepoName}`); + } + + const rel = path.relative(cloneRoot, safeTarget); if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { - throw new Error(`Clone target must be a subdirectory of ${CLONE_ROOT}`); + throw new Error(`Clone target must be a subdirectory of ${cloneRoot}`); } // Always validate the requested URL — the prior shape only ran this in // the code path where the repo was cloned. Now it runs unconditionally, // preventing SSRF / blocked-host bypasses even when targetDir already exists. validateGitUrl(url); + await fs.mkdir(cloneRoot, { recursive: true }); + if (options?.allowedCloneRoot) { + await assertDirectoryOwnerAndPermissions(cloneRoot); + } + await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertPreRealpathContainment(cloneRoot, safeTarget); const exists = await fs.access(path.join(safeTarget, '.git')).then( () => true, () => false, ); + const targetExists = await fs.access(safeTarget).then( + () => true, + () => false, + ); + if (exists) { + await assertPostRealpathContainment(cloneRoot, safeTarget); // Confirm the existing clone is actually the same repository the caller // requested. Without this check, a pull would silently succeed against // whatever remote the dir was originally cloned from. @@ -480,14 +513,72 @@ export async function cloneOrPull( onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url }); } else { + if (targetExists) { + throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); + } await fs.mkdir(path.dirname(safeTarget), { recursive: true }); + await assertNoSymlinkPath(cloneRoot, safeTarget); + await assertPreRealpathContainment(cloneRoot, safeTarget); onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); - await runGit(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + try { + const runGitImpl = options?.runGitForTest ?? runGit; + await runGitImpl(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + await assertPostRealpathContainment(cloneRoot, safeTarget); + } catch (err: unknown) { + if (options?.quarantineRoot) { + await fs + .access(safeTarget) + .then(async () => { + await quarantineAutoSyncPartial(safeTarget, options.quarantineRoot!); + }) + .catch(() => {}); + } + throw err; + } } return safeTarget; } +async function assertPreRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realParent = await fs.realpath(path.dirname(target)); + const parentRel = path.relative(realRoot, realParent); + if (parentRel.startsWith('..') || path.isAbsolute(parentRel)) { + throw new Error(`Clone target parent must resolve inside ${root}`); + } +} + +async function assertPostRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realTarget = await fs.realpath(target); + const rel = path.relative(realRoot, realTarget); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { + throw new Error(`Clone target must resolve inside ${root}`); + } +} + +async function assertNoSymlinkPath(root: string, target: string): Promise { + const resolvedRoot = path.resolve(root); + const resolvedTarget = path.resolve(target); + const relativeTarget = path.relative(resolvedRoot, resolvedTarget); + if (relativeTarget.startsWith('..') || path.isAbsolute(relativeTarget)) return; + let current = resolvedRoot; + for (const segment of relativeTarget.split(path.sep).filter(Boolean)) { + current = path.join(current, segment); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlink in clone target path: ${current}`); + } + } +} + /** * Hosts the per-request GitHub PAT may be sent to. Exported so the * /api/analyze boundary check and this injection-site check share one diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts new file mode 100644 index 000000000..59e358c6a --- /dev/null +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -0,0 +1,421 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; + +import { + addRepoToGroup, + getConfiguredRepoPath, + maybeStartAutoSyncFromEnv, + runAutoSyncOnce, +} from '../../src/core/auto-sync/index.js'; +import type { AutoSyncConfig, AutoSyncRunDeps } from '../../src/core/auto-sync/index.js'; + +const config: AutoSyncConfig = { + configPath: '/tmp/.gitnexus/sync_config.yml', + syncIntervalMinutes: 10, + projects: [ + { + localPath: '/tmp/repos', + gitnexusGroup: 'back_end', + branches: ['master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }, + ], +}; + +const cloneRoot = { + root: '/tmp/repos', + quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRetentionDays: 14, +}; + +function withCloneRoot(deps: Partial): Partial { + return { + resolveCloneRoot: vi.fn(async () => cloneRoot), + ...deps, + }; +} + +describe('auto-sync runner', () => { + it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/quarantine', + }, + ); + expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); + expect(deps.runFullAnalysis).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + { branch: 'master', skipAgentsMd: true, skipSkills: true }, + { onProgress: expect.any(Function) }, + ); + expect(deps.registerRepo).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + expect.objectContaining({ lastCommit: 'commit-2', branch: 'master' }), + { name: 'qts_account', allowDuplicateName: true }, + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('skips analysis when commit id has not changed', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-1'), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(0); + expect(result.skippedAnalysis).toBe(1); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.syncGroupByName).not.toHaveBeenCalled(); + }); + + it('uses local_path plus repo name as the clone target', async () => { + expect(getConfiguredRepoPath(config.projects[0], 'qts_account')).toBe('/tmp/repos/qts_account'); + + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/quarantine', + }, + ); + }); + + it('skips analysis when the checked out branch is not configured', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch develop is not in configured branches: master.', + ); + }); + + it('skips analysis when the checked out repository is detached', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => undefined), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch is not in configured branches: master.', + ); + }); + + it('isolates repository, analysis, and group sync failures', async () => { + const errorLogger = vi.fn(); + const failingConfig: AutoSyncConfig = { + ...config, + projects: [ + { + ...config.projects[0], + remoteUrls: [ + 'git@gitee.com:qts_server/failing_sync.git', + 'git@gitee.com:qts_server/qts_account.git', + ], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + if (remoteUrl.includes('failing_sync')) throw new Error('sync failed'); + return '/tmp/repos/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => { + throw new Error('analysis failed'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => { + throw new Error('group sync failed'); + }), + }); + + const result = await runAutoSyncOnce(failingConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 3 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.registerRepo).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(failingConfig.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + codeCommitId: 'commit-2', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Repository sync failed for git@gitee.com:qts_server/failing_sync.git'), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Group sync failed for back_end'), + ); + }); + + it('detects existing groupPath to registryName mappings as already joined', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-group-')); + try { + process.env.GITNEXUS_HOME = tempDir; + const groupDir = path.join(tempDir, 'groups', 'back_end'); + await fs.mkdir(groupDir, { recursive: true }); + await fs.writeFile( + path.join(groupDir, 'group.yaml'), + [ + 'version: 1', + 'name: back_end', + 'repos:', + ' hr/hiring/backend: qts_account', + ].join('\n'), + ); + + await expect(addRepoToGroup({ gitnexusGroup: 'back_end' }, 'qts_account')).resolves.toBe( + false, + ); + + await expect(fs.readFile(path.join(groupDir, 'group.yaml'), 'utf-8')).resolves.toContain( + 'hr/hiring/backend: qts_account', + ); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); + +describe('auto-sync starter', () => { + it('does not read config or register timers when the flag is disabled', async () => { + const previous = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '0'; + const setIntervalFn = vi.fn() as unknown as typeof setInterval; + + try { + const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn }); + expect(handle).toBeNull(); + expect(setIntervalFn).not.toHaveBeenCalled(); + } finally { + if (previous === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previous; + } + }); + + it('registers a clearable timer when enabled with a valid config', async () => { + const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + const runOnce = vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); + + try { + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'sync_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' gitnexus_group: back_end', + ' branch: master', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn, clearIntervalFn, runOnce }); + + expect(handle).not.toBeNull(); + expect(runOnce).toHaveBeenCalledTimes(1); + expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); + expect(timer.unref).toHaveBeenCalled(); + + handle?.stop(); + + expect(clearIntervalFn).toHaveBeenCalledWith(timer); + } finally { + if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('skips overlapping scheduled runs while a previous run is active', async () => { + const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + let scheduled: (() => void) | undefined; + const setIntervalFn = vi.fn((fn: () => void) => { + scheduled = fn; + return timer; + }) as unknown as typeof setInterval; + const stderr = { write: vi.fn() }; + let releaseRun: (() => void) | undefined; + const runOnce = vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }), + ); + + try { + process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'sync_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - https://example.com/team/repo.git', + ].join('\n'), + ); + + await maybeStartAutoSyncFromEnv({ setIntervalFn, runOnce, stderr }); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(1); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Previous run is still active; skipping overlapping run.\n', + ); + + releaseRun?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(2); + } finally { + if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts new file mode 100644 index 000000000..6cc636ba6 --- /dev/null +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -0,0 +1,250 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + extractRepoNameFromRemoteUrl, + loadAutoSyncConfig, + parseAutoSyncFlag, + parseBranchCandidates, + resolveConfiguredCloneRoot, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, +} from '../../src/core/auto-sync/index.js'; + +describe('auto-sync', () => { + let tempDir: string; + let gitnexusHome: string; + let oldHome: string | undefined; + let oldFlag: string | undefined; + + beforeEach(async () => { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + tempDir = await fs.realpath(await fs.mkdtemp(path.join(base, 'gitnexus-auto-sync-'))); + gitnexusHome = path.join(tempDir, '.gitnexus'); + await fs.mkdir(gitnexusHome); + oldHome = process.env.GITNEXUS_HOME; + oldFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + process.env.GITNEXUS_HOME = gitnexusHome; + delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + }); + + afterEach(async () => { + if (oldHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = oldHome; + if (oldFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = oldFlag; + await fs.rm(tempDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('keeps auto sync disabled when the flag is unset or 0', () => { + expect(parseAutoSyncFlag(undefined)).toEqual({ enabled: false, reason: 'unset' }); + expect(parseAutoSyncFlag('0')).toEqual({ enabled: false, reason: 'disabled' }); + }); + + it('enables auto sync only for the explicit value 1', () => { + expect(parseAutoSyncFlag('1')).toEqual({ enabled: true }); + expect(parseAutoSyncFlag('true')).toEqual({ + enabled: false, + reason: 'invalid', + message: '[auto-sync] AUTO_UPDATE_AND_ANALYZE_FLAG must be 0 or 1; got "true". Auto sync is disabled.', + }); + }); + + it('loads sync_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'sync_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' gitnexus_group: back_end', + ' branch: test, master, test', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config to load'); + expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'sync_config.yml')); + expect(loaded.config.syncIntervalMinutes).toBe(10); + expect(loaded.config.projects[0]).toMatchObject({ + localPath: '/tmp/repos', + gitnexusGroup: 'back_end', + branches: ['test', 'master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }); + }); + + it('reports missing config without throwing', async () => { + const loaded = await loadAutoSyncConfig(); + + expect(loaded).toEqual({ + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'sync_config.yml')}. Auto sync is skipped.`, + }); + }); + + it('reports invalid config without throwing', async () => { + await fs.writeFile(path.join(gitnexusHome, 'sync_config.yml'), 'projects: []\n'); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.reason).toBe('invalid'); + expect(loaded.message).toContain('[auto-sync] Invalid sync_config.yml:'); + expect(loaded.message).toContain('sync_interval_minutes must be a positive integer'); + expect(loaded.message).toContain('projects must contain at least one project'); + }); + + it('rejects missing, relative, and traversal local_path values at config load', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'sync_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: ../repos', + ' branch: master', + ' remote_urls:', + ' - https://example.com/team/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('local_path must be an absolute path'); + }); + + it('hard-fails unsafe configured clone roots', async () => { + await expect(resolveConfiguredCloneRoot('/')).rejects.toThrow('unsafe auto-sync clone root'); + await expect(resolveConfiguredCloneRoot(os.homedir())).rejects.toThrow('unsafe auto-sync clone root'); + await expect(resolveConfiguredCloneRoot(path.join(await fs.realpath(os.tmpdir()), 'repos'))).rejects.toThrow( + 'unsafe auto-sync clone root', + ); + const root = path.join(tempDir, 'repos'); + await expect(resolveConfiguredCloneRoot(`${root}/../repos`)).rejects.toThrow( + 'normalized', + ); + }); + + it('rejects GitNexus internal directory descendants as clone roots', async () => { + for (const internalDir of ['groups', 'indexes', 'quarantine']) { + const root = path.join(gitnexusHome, internalDir, 'repo-root'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('GitNexus internal directory'); + } + }); + + it('rejects symlinks in configured clone root paths', async () => { + const realRoot = path.join(tempDir, 'real-root'); + const linkRoot = path.join(tempDir, 'link-root'); + await fs.mkdir(realRoot); + await fs.symlink(realRoot, linkRoot); + + await expect(resolveConfiguredCloneRoot(linkRoot)).rejects.toThrow('symlink'); + }); + + it('resolves safe configured clone roots and reports quarantine retention', async () => { + const root = path.join(tempDir, 'repos'); + await fs.mkdir(root); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'quarantine'), + quarantineRetentionDays: 14, + }), + ); + }); + + it('parses branch strings and arrays with trimming and de-duplication', () => { + expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']); + expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']); + }); + + it('extracts safe repository names from remote URLs', () => { + expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( + 'qts_account', + ); + expect(extractRepoNameFromRemoteUrl('https://example.com/team/repo-name.git')).toBe( + 'repo-name', + ); + }); + + it('rejects unsafe repository names without sanitizing them', () => { + expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/repo$name.git')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/..')).toThrow( + 'valid repository name', + ); + }); + + it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => { + expect(shouldAnalyzeCommit({ currentCommit: 'abc', previousAnalyzedCommit: 'abc' })).toBe( + false, + ); + expect( + shouldAnalyzeCommit({ + currentCommit: 'abc', + previousAnalyzedCommit: 'abc', + previousStatus: 'failed', + }), + ).toBe(true); + expect(shouldAnalyzeCommit({ currentCommit: 'def', previousAnalyzedCommit: 'abc' })).toBe( + true, + ); + }); + + it('saves state atomically and reloads it', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + + await saveAutoSyncState( + { + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }, + statePath, + ); + + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('auto-sync-state.json.tmp'), + ); + await expect(loadAutoSyncState(statePath)).resolves.toEqual({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }); + }); + + it('returns empty state and reports corrupt state files', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + await fs.writeFile(statePath, '{not-json', 'utf-8'); + + await expect(loadAutoSyncState(statePath)).resolves.toEqual({}); + + expect(stderr).toHaveBeenCalledWith( + `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + }); +}); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 4db09c837..9d879a22a 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -42,6 +42,12 @@ import { getGlobalDir } from '../../src/storage/repo-manager.js'; // load, the same point CLONE_ROOT is frozen, so the two always agree. const EXPECTED_CLONE_ROOT = path.resolve(path.join(getGlobalDir(), 'repos')); +async function mkControlledRoot(prefix: string): Promise { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + return fs.realpath(await fs.mkdtemp(path.join(base, prefix))); +} + describe('git-clone', () => { describe('extractRepoName', () => { it('extracts name from HTTPS URL', () => { @@ -95,29 +101,39 @@ describe('git-clone', () => { expect(elapsedMs).toBeLessThan(500); }); - it('strips leading dashes to prevent argument injection', () => { - expect(extractRepoName('https://github.com/user/--upload-pack=payload.git')).toBe( - 'upload-pack_payload', + it('rejects leading dashes to prevent argument injection', () => { + expect(() => extractRepoName('https://github.com/user/--upload-pack=payload.git')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://github.com/user/-repo')).toThrow( + 'valid repository name', ); - expect(extractRepoName('https://github.com/user/-repo')).toBe('repo'); }); - it('sanitizes unsafe directory characters', () => { - // sanitizeRepoName turns into _tag_ - expect(extractRepoName('https://github.com/user/repo.git')).toBe('repo_tag_'); + it('rejects unsafe directory characters instead of sanitizing them', () => { + expect(() => extractRepoName('https://github.com/user/repo.git')).toThrow( + 'valid repository name', + ); }); - it('sanitizes shell metacharacters in URL segments', () => { + it('rejects shell metacharacters in URL segments', () => { // The split on /[/:]/ does not split on backslashes or other shell chars, - // so a name like `repo;rm -rf /` would slip through without the pattern. - // After fix/sanitize-repo-name, these are sanitized to underscores. - expect(extractRepoName('https://example.com/foo:repo;rm')).toBe('repo_rm'); - expect(extractRepoName('https://example.com/foo:repo$x')).toBe('repo_x'); + // so a name like `repo;rm -rf /` must fail instead of being rewritten. + expect(() => extractRepoName('https://example.com/foo:repo;rm')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo$x')).toThrow( + 'valid repository name', + ); }); - it('sanitizes whitespace and backslashes', () => { - expect(extractRepoName('https://example.com/foo:repo name')).toBe('repo_name'); - expect(extractRepoName('https://example.com/foo:repo\\name')).toBe('repo_name'); + it('rejects whitespace and backslashes', () => { + expect(() => extractRepoName('https://example.com/foo:repo name')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo\\name')).toThrow( + 'valid repository name', + ); }); }); @@ -534,6 +550,154 @@ describe('git-clone', () => { 'Only https:// and http://', ); }); + + it('allows an explicitly controlled auto-sync clone root outside the default root', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + const target = path.join(root, 'repo'); + await expect( + cloneOrPull('http://127.0.0.1/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('private/internal'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled-root target names that do not match the remote repo name', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'other'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('basename must match'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects symlink children before clone or pull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const outside = await mkControlledRoot('gitnexus-outside-'); + try { + await fs.symlink(outside, path.join(root, 'repo')); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('symlink'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects existing clones whose remote origin mismatches the requested URL', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + try { + await new Promise((resolve, reject) => { + const proc = spawn('git', ['init'], { cwd: root, stdio: 'ignore' }); + proc.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`git init ${code}`)))); + proc.on('error', reject); + }); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')).catch(async () => { + await fs.mkdir(target); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')); + }); + await fs.writeFile( + path.join(target, '.git', 'config'), + [ + '[remote "origin"]', + '\turl = https://example.com/other/repo.git', + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '', + ].join('\n'), + ); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('not the requested URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('quarantines partial auto-sync clone output on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + runGitForTest: async () => { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'partial.txt'), 'partial', 'utf-8'); + throw new Error('git clone failed (exit code 128)'); + }, + }), + ).rejects.toThrow('git clone failed'); + + const entries = await fs.readdir(quarantineRoot); + expect(entries.some((entry) => entry.startsWith('auto-sync-') && entry.endsWith('-repo'))).toBe( + true, + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('does not quarantine an existing non-git directory on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'user-file.txt'), 'keep me', 'utf-8'); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + }), + ).rejects.toThrow('already exists but is not a git repository'); + + await expect(fs.readFile(path.join(target, 'user-file.txt'), 'utf-8')).resolves.toBe( + 'keep me', + ); + await expect(fs.access(quarantineRoot)).rejects.toThrow(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled clone roots with unsafe permissions inside cloneOrPull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await fs.chmod(root, 0o777); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(root, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); }); describe('isAzureDevOpsUrl', () => {