diff --git a/README.md b/README.md index f19ad996c..dbb203b64 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16, auto-sized to the repo). 0 is rejected — there is no sequential mode. +gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus list # List all indexed repositories @@ -262,6 +263,27 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +### `gitnexus watch` + +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + group_name: back_end + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped by `floor(availableMemoryGB / 2)` with a minimum of `1`, printed at each loop start. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group processing. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. diff --git a/gitnexus/README.md b/gitnexus/README.md index 62a329b75..2f30a5e84 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -169,6 +169,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) +gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus index # Register an existing .gitnexus/ folder into the global registry @@ -197,6 +198,27 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +### `gitnexus watch` + +`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + group_name: back_end + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project. `GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `GITNEXUS_HOME/watch/auto-sync-state.json`. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index def76601b..f947c364c 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -13,6 +13,7 @@ const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', uninstall: 'help.command.uninstall.description', + watch: 'help.command.watch.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', serve: 'help.command.serve.description', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index caf61ab83..6144ef7b1 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -120,6 +120,8 @@ export const en = { 'One-time setup: configure MCP for Cursor, Claude Code, OpenCode, Codex', 'help.command.uninstall.description': 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + 'help.command.watch.description': + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 01abeedc1..627aae178 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -122,6 +122,8 @@ export const zhCN = { 'help.command.setup.description': '一次性设置：为 Cursor、Claude Code、OpenCode、Codex 配置 MCP', 'help.command.uninstall.description': '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', + 'help.command.watch.description': + '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 4426eec9d..37a527728 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -15,12 +15,6 @@ const _require = createRequire(import.meta.url); const pkg = _require('../../package.json'); const program = new Command(); -if (process.env.AUTO_UPDATE_AND_ANALYZE_FLAG?.trim() === '1') { - void import('../core/auto-sync/index.js').then(({ maybeStartAutoSyncFromEnv }) => - maybeStartAutoSyncFromEnv(), - ); -} - function collectCodingAgents(value: string, previous: string[] | undefined): string[] { return [...(previous ?? []), ...value.split(',')]; } @@ -47,6 +41,25 @@ program .option('-f, --force', 'Apply the changes (default is a dry-run preview)') .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); +program + .command('watch [action]') + .description( + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + ) + .addHelpText( + 'after', + [ + '', + 'Actions: init, start (default), restart, stop, status', + 'Configuration: GITNEXUS_HOME/watch_config.yml', + 'Runtime files: GITNEXUS_HOME/watch/watch.pid, watch.lock, watch.status.json, auto-sync-state.json', + 'Writes: GITNEXUS_HOME/watch/project_commit_info.txt', + 'Remote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.', + 'Runs once immediately, then repeats on sync_interval_minutes.', + ].join('\n'), + ) + .action(createLazyAction(() => import('./watch.js'), 'watchCommand')); + // Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook // before it overwrites the var, so the postAction hook can restore it. The // analyzeCommand env snapshot is taken AFTER this hook runs, so it cannot undo diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts index 9c0a67e99..febf8490f 100644 --- a/gitnexus/src/core/auto-sync/config.ts +++ b/gitnexus/src/core/auto-sync/config.ts @@ -7,14 +7,18 @@ import { normalizeConfiguredCloneRoot } from './path-security.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -export const AUTO_SYNC_FLAG = 'AUTO_UPDATE_AND_ANALYZE_FLAG'; -export const AUTO_SYNC_CONFIG_FILE = 'sync_config.yml'; +export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml'; const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; const MIN_SYNC_INTERVAL_MINUTES = 5; +const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_CONCURRENCY = 1; +export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3; +const MIN_ANALYZE_FAILURE_THRESHOLD = 2; +const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']); export interface AutoSyncProjectConfig { localPath: string; - gitnexusGroup?: string; + groupName?: string; branches: string[]; remoteUrls: string[]; } @@ -22,29 +26,16 @@ export interface AutoSyncProjectConfig { export interface AutoSyncConfig { configPath: string; syncIntervalMinutes: number; + repoGitTimeoutMs: number; + maxConcurrency: number; + analyzeFailureThreshold: number; projects: AutoSyncProjectConfig[]; } -export type AutoSyncFlagDecision = - | { enabled: true } - | { enabled: false; reason: 'unset' | 'disabled' | 'invalid'; message?: string }; - export type AutoSyncConfigLoadResult = | { ok: true; config: AutoSyncConfig } | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; -export function parseAutoSyncFlag(raw = process.env[AUTO_SYNC_FLAG]): AutoSyncFlagDecision { - if (raw === undefined || raw.trim() === '') return { enabled: false, reason: 'unset' }; - const trimmed = raw.trim(); - if (trimmed === '0') return { enabled: false, reason: 'disabled' }; - if (trimmed === '1') return { enabled: true }; - return { - enabled: false, - reason: 'invalid', - message: `[auto-sync] ${AUTO_SYNC_FLAG} must be 0 or 1; got "${trimmed}". Auto sync is disabled.`, - }; -} - export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); } @@ -92,7 +83,7 @@ export async function loadAutoSyncConfig( return { ok: false, reason: 'invalid', - message: `[auto-sync] Invalid sync_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + message: `[auto-sync] Invalid watch_config.yml: ${(err as Error).message}. Auto sync is skipped.`, }; } } @@ -111,6 +102,31 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); } + const maxConcurrency = + raw.max_concurrency === undefined ? DEFAULT_MAX_CONCURRENCY : Number(raw.max_concurrency); + if (!Number.isInteger(maxConcurrency) || maxConcurrency <= 0) { + errors.push('max_concurrency must be a positive integer'); + } + + const repoGitTimeoutMs = + raw.repo_git_timeout === undefined + ? DEFAULT_REPO_GIT_TIMEOUT_MS + : parseDurationMs(raw.repo_git_timeout); + if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) { + errors.push('repo_git_timeout must be a positive duration such as 10s'); + } + + const analyzeFailureThreshold = + raw.analyze_failure_threshold === undefined + ? DEFAULT_ANALYZE_FAILURE_THRESHOLD + : Number(raw.analyze_failure_threshold); + if ( + !Number.isInteger(analyzeFailureThreshold) || + analyzeFailureThreshold < MIN_ANALYZE_FAILURE_THRESHOLD + ) { + errors.push(`analyze_failure_threshold must be an integer >= ${MIN_ANALYZE_FAILURE_THRESHOLD}`); + } + const rawProjects = raw.projects; if (!Array.isArray(rawProjects) || rawProjects.length === 0) { errors.push('projects must contain at least one project'); @@ -142,22 +158,86 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy if (remoteUrls.length === 0) { errors.push(`projects[${index}].remote_urls must contain at least one URL`); } + for (let urlIndex = 0; urlIndex < remoteUrls.length; urlIndex += 1) { + try { + validateAutoSyncRemoteUrl(remoteUrls[urlIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].remote_urls[${urlIndex}] ${(err as Error).message}`); + } + } - const branches = parseBranchCandidates(project.branch); - if (branches.length === 0) errors.push(`projects[${index}].branch is required`); + if (project.branch !== undefined && project.branches !== undefined) { + errors.push(`projects[${index}] must not set both branch and branches`); + } + const branches = parseBranchCandidates( + project.branches !== undefined ? project.branches : project.branch, + ); + if (branches.length === 0) errors.push(`projects[${index}].branches is required`); + for (let branchIndex = 0; branchIndex < branches.length; branchIndex += 1) { + try { + validateAutoSyncBranchName(branches[branchIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].branches[${branchIndex}] ${(err as Error).message}`); + } + } - const gitnexusGroup = - typeof project.gitnexus_group === 'string' ? project.gitnexus_group.trim() : undefined; - if (gitnexusGroup && !GROUP_NAME_PATTERN.test(gitnexusGroup)) { - errors.push(`projects[${index}].gitnexus_group is invalid`); + const groupName = + typeof project.group_name === 'string' && project.group_name.trim() + ? project.group_name.trim() + : undefined; + if (groupName && !GROUP_NAME_PATTERN.test(groupName)) { + errors.push(`projects[${index}].group_name is invalid`); } if (localPath && remoteUrls.length > 0 && branches.length > 0) { - projects.push({ localPath, gitnexusGroup, branches, remoteUrls }); + projects.push({ localPath, groupName, branches, remoteUrls }); } }); } if (errors.length > 0) throw new Error(errors.join('; ')); - return { configPath, syncIntervalMinutes: interval, projects }; + return { + configPath, + syncIntervalMinutes: interval, + repoGitTimeoutMs, + maxConcurrency, + analyzeFailureThreshold, + projects, + }; +} + +export function validateAutoSyncRemoteUrl(remoteUrl: string): void { + const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim()); + if (!match) { + throw new Error('must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git'); + } + const host = match[1].toLowerCase(); + const repoPath = match[2]; + if (!ALLOWED_REMOTE_HOSTS.has(host)) { + throw new Error('host must be one of github.com, gitlab.com, or gitee.com'); + } + if (repoPath.startsWith('/') || repoPath.includes('..') || repoPath.split('/').length < 2) { + throw new Error('path must include owner/repo without traversal'); + } +} + +export function validateAutoSyncBranchName(branch: string): void { + if (!branch.trim()) throw new Error('must not be empty'); + if (/[\s\0-\x1f\x7f]/.test(branch)) throw new Error('must not contain whitespace or control characters'); + if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref'); + if (branch.startsWith('-')) throw new Error('must not start with "-"'); + if (branch.includes('..')) throw new Error('must not contain ".."'); + if (branch.includes('`')) throw new Error('must not contain backticks'); +} + +export function parseDurationMs(value: unknown): number { + if (typeof value === 'number') return value * 1_000; + const raw = String(value ?? '').trim(); + const match = /^(\d+)(ms|s|m)?$/.exec(raw); + if (!match) return Number.NaN; + const amount = Number(match[1]); + const unit = match[2] ?? 's'; + if (unit === 'ms') return amount; + if (unit === 's') return amount * 1_000; + return amount * 60_000; } diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts index de1c0b4f7..6dc23a177 100644 --- a/gitnexus/src/core/auto-sync/index.ts +++ b/gitnexus/src/core/auto-sync/index.ts @@ -1,25 +1,29 @@ export { AUTO_SYNC_CONFIG_FILE, - AUTO_SYNC_FLAG, getAutoSyncConfigPath, loadAutoSyncConfig, parseAutoSyncConfig, - parseAutoSyncFlag, parseBranchCandidates, + parseDurationMs, + validateAutoSyncBranchName, + validateAutoSyncRemoteUrl, type AutoSyncConfig, type AutoSyncConfigLoadResult, - type AutoSyncFlagDecision, type AutoSyncProjectConfig, } from './config.js'; export { buildStateKey, + getAutoSyncWatchDir, getAutoSyncStatePath, + getProjectCommitInfoPath, loadAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, + writeProjectCommitInfo, type AutoSyncAnalyzeStatus, type AutoSyncCommitState, type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, } from './state.js'; export { extractRepoNameFromRemoteUrl } from './repo.js'; export { @@ -31,10 +35,19 @@ export { export { addRepoToGroup, getConfiguredRepoPath, + resolveActualConcurrency, runAutoSyncOnce, syncGroupByName, type AutoSyncLogger, type AutoSyncRunDeps, type AutoSyncRunResult, } from './runner.js'; -export { maybeStartAutoSyncFromEnv, type AutoSyncStartHandle } from './starter.js'; +export { + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + startAutoSyncWatch, + stopAutoSyncWatch, + type AutoSyncStartHandle, + type AutoSyncWatchPaths, + type WatchStatusRecord, +} from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts index 10f5d6ab7..e8510c248 100644 --- a/gitnexus/src/core/auto-sync/path-security.ts +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; +import { getAutoSyncWatchDir } from './state.js'; const DANGEROUS_ROOTS = new Set( [ @@ -61,6 +62,7 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise number; } export interface AutoSyncRunResult { @@ -59,9 +65,11 @@ const DEFAULT_DEPS: AutoSyncRunDeps = { registerRepo, loadState: loadAutoSyncState, saveState: saveAutoSyncState, + writeCommitInfo: writeProjectCommitInfo, addRepoToGroup, syncGroupByName, resolveCloneRoot: resolveConfiguredCloneRoot, + getAvailableMemoryGB: () => Math.floor(process.availableMemory?.() ?? 0) / 1024 / 1024 / 1024, }; export async function runAutoSyncOnce( @@ -74,89 +82,192 @@ export async function runAutoSyncOnce( const state = await deps.loadState(); const groupsToSync = new Set(); const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + const commitInfoEntries: ProjectCommitInfoEntry[] = []; + const actualConcurrency = resolveActualConcurrency(config.maxConcurrency, deps.getAvailableMemoryGB()); + logger.info( + `[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`, + ); - for (const project of config.projects) { - for (const remoteUrl of project.remoteUrls) { - try { - const repoName = extractRepoNameFromRemoteUrl(remoteUrl); - const cloneRoot = await deps.resolveCloneRoot(project.localPath); - const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); - await deps.cloneOrPull(remoteUrl, targetDir, undefined, { - allowedCloneRoot: cloneRoot.root, - expectedRepoName: repoName, - quarantineRoot: cloneRoot.quarantineRoot, - }); - result.synced += 1; - - const currentBranch = deps.getCurrentBranch(targetDir); - if (!currentBranch || !project.branches.includes(currentBranch)) { - result.skippedAnalysis += 1; - logger.warn( - `[auto-sync] Skip analysis for ${targetDir}; current branch ${currentBranch ?? ''} is not in configured branches: ${project.branches.join(', ')}.`, - ); - continue; - } - - const branch = currentBranch; - const currentCommit = deps.getCurrentCommit(targetDir); - const stateKey = buildStateKey(targetDir, branch); - const previous = state[stateKey]; - let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; - let analyzedCommitId = previous?.analyzedCommitId; - - if ( - shouldAnalyzeCommit({ - currentCommit, - previousAnalyzedCommit: previous?.analyzedCommitId, - previousStatus: previous?.lastAnalyzeStatus, - }) - ) { - try { - const analysis = await deps.runFullAnalysis( - targetDir, - { branch, skipAgentsMd: true, skipSkills: true }, - { onProgress: () => {} }, - ); - const meta: RepoMeta = { - repoPath: targetDir, - lastCommit: currentCommit, - indexedAt: now().toISOString(), - stats: analysis.stats, - branch, - }; - await deps.registerRepo(targetDir, meta, { name: repoName, allowDuplicateName: true }); - analyzeStatus = 'success'; - analyzedCommitId = currentCommit; - result.analyzed += 1; - } catch (err: unknown) { - analyzeStatus = 'failed'; - result.failed += 1; - logger.error(`[auto-sync] Analysis failed for ${targetDir}: ${(err as Error).message}`); - } - } else { - result.skippedAnalysis += 1; - logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); - } - - state[stateKey] = { - codeCommitId: currentCommit, - analyzedCommitId, - lastAnalyzeStatus: analyzeStatus, - lastSyncTime: now().toISOString(), + const workItems = await buildWorkItems(config, deps); + const repoResults = await mapWithConcurrency(workItems, actualConcurrency, async (item) => { + const lastSyncTime = now().toISOString(); + try { + validateAutoSyncRemoteUrl(item.remoteUrl); + const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName); + const syncResult = await syncFirstAvailableBranch({ + item, + repoName, + targetDir, + timeoutMs: config.repoGitTimeoutMs, + deps, + logger, + }); + if (syncResult.ok === false) { + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir, + branch: item.project.branches[0], + status: syncResult.status, + analyzeConsecutiveFailures: 0, + lastSyncTime, }; + } - if (project.gitnexusGroup) { - const added = await deps.addRepoToGroup(project, repoName); - if (added) groupsToSync.add(project.gitnexusGroup); + const currentBranch = syncResult.branch; + + const currentCommit = deps.getCurrentCommit(targetDir); + const stateKey = buildStateKey(targetDir, currentBranch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; + let lastAnalyzeError = previous?.lastAnalyzeError; + let stats: RepoMeta['stats'] | undefined; + + if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { + analyzeStatus = 'threshold_skipped'; + logger.error( + `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, + ); + } else if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runFullAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + stats = analysis.stats; + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } catch (err: unknown) { + analyzeStatus = 'failed'; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = shortErrorMessage(err); + logger.error( + `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, + ); } + } else { + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + + return { + kind: 'synced' as const, + project: item.project, + repoName, + remoteUrl: item.remoteUrl, + targetDir, + branch: currentBranch, + currentCommit, + analyzedCommitId, + analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + stats, + stateKey, + lastSyncTime, + }; + } catch (err: unknown) { + logger.error(`[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir: '', + status: 'sync_failed' as const, + lastSyncTime, + }; + } + }); + + for (const repoResult of repoResults) { + if (repoResult.kind === 'failed') { + result.failed += 1; + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + status: repoResult.status, + lastSyncTime: repoResult.lastSyncTime, + }); + continue; + } + + result.synced += 1; + const stateEntry: AutoSyncCommitStateEntry = { + codeCommitId: repoResult.currentCommit, + analyzedCommitId: repoResult.analyzedCommitId, + lastAnalyzeStatus: repoResult.analyzeStatus, + analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, + lastAnalyzeError: repoResult.lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }; + state[repoResult.stateKey] = stateEntry; + if (repoResult.analyzeStatus === 'success') { + const meta: RepoMeta = { + repoPath: repoResult.targetDir, + lastCommit: repoResult.currentCommit, + indexedAt: repoResult.lastSyncTime, + stats: repoResult.stats!, + branch: repoResult.branch, + }; + await deps.registerRepo(repoResult.targetDir, meta, { + name: repoResult.repoName, + allowDuplicateName: true, + }); + result.analyzed += 1; + } else if (repoResult.analyzeStatus === 'failed') { + result.failed += 1; + } else if (repoResult.analyzeStatus === 'threshold_skipped') { + result.skippedAnalysis += 1; + } else { + result.skippedAnalysis += 1; + } + + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + codeCommitId: repoResult.currentCommit, + analyzedCommitId: repoResult.analyzedCommitId, + status: repoResult.analyzeStatus, + analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures, + analyzeFailureThreshold: config.analyzeFailureThreshold, + lastAnalyzeError: repoResult.lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }); + + if (repoResult.project.groupName) { + let groupMembershipOk = false; + try { + await deps.addRepoToGroup(repoResult.project, repoResult.repoName); + groupMembershipOk = true; } catch (err: unknown) { result.failed += 1; - logger.error(`[auto-sync] Repository sync failed for ${remoteUrl}: ${(err as Error).message}`); + logger.error(`[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`); + } + if (groupMembershipOk && repoResult.analyzeStatus === 'success') { + groupsToSync.add(repoResult.project.groupName); } } } await deps.saveState(state); + await deps.writeCommitInfo(commitInfoEntries); for (const groupName of groupsToSync) { try { await deps.syncGroupByName(groupName); @@ -168,6 +279,11 @@ export async function runAutoSyncOnce( return result; } +function shortErrorMessage(err: unknown): string { + const message = (err as Error).message || String(err); + return message.replace(/\s+/g, ' ').slice(0, 240); +} + export function getConfiguredRepoPath( project: Pick, repoName: string, @@ -176,11 +292,11 @@ export function getConfiguredRepoPath( } export async function addRepoToGroup( - project: Pick, + project: Pick, repoName: string, ): Promise { - if (!project.gitnexusGroup) return false; - const groupDir = getGroupDir(getDefaultGitnexusDir(), project.gitnexusGroup); + if (!project.groupName) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName); const config = await loadGroupConfig(groupDir); if (Object.values(config.repos).includes(repoName)) return false; config.repos[repoName] = repoName; @@ -199,3 +315,100 @@ async function writeGroupConfigAtomic(filePath: string, config: unknown): Promis await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); await fs.rename(tmpPath, filePath); } + +export function resolveActualConcurrency(configured: number, availableMemoryGB: number): number { + const memoryLimit = Math.max(1, Math.floor(availableMemoryGB / 2)); + return Math.max(1, Math.min(configured, memoryLimit)); +} + +async function buildWorkItems(config: AutoSyncConfig, deps: AutoSyncRunDeps): Promise { + const items: AutoSyncWorkItem[] = []; + const targetOwners = new Map(); + for (const project of config.projects) { + const cloneRoot = await deps.resolveCloneRoot(project.localPath); + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName); + const previous = targetOwners.get(targetDir); + if (previous !== undefined) { + throw new Error(`Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`); + } + targetOwners.set(targetDir, remoteUrl); + } catch (err: unknown) { + if ((err as Error).message.startsWith('Duplicate auto-sync targetDir')) throw err; + } + items.push({ project, remoteUrl, cloneRoot }); + } + } + return items; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let nextIndex = 0; + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (nextIndex < items.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await worker(items[currentIndex]); + } + }); + await Promise.all(runners); + return results; +} + +interface AutoSyncWorkItem { + project: AutoSyncProjectConfig; + remoteUrl: string; + cloneRoot: Awaited>; +} + +async function syncFirstAvailableBranch(input: { + item: AutoSyncWorkItem; + repoName: string; + targetDir: string; + timeoutMs: number; + deps: AutoSyncRunDeps; + logger: AutoSyncLogger; +}): Promise< + | { ok: true; branch: string } + | { ok: false; status: 'branch_unavailable' | 'sync_timeout'; message: string } +> { + const failures: string[] = []; + let sawTimeout = false; + for (const branch of input.item.project.branches) { + try { + await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, { + allowedCloneRoot: input.item.cloneRoot.root, + expectedRepoName: input.repoName, + quarantineRoot: input.item.cloneRoot.quarantineRoot, + allowAutoSyncSsh: true, + timeoutMs: input.timeoutMs, + branch, + }); + const currentBranch = input.deps.getCurrentBranch(input.targetDir); + if (currentBranch === branch) return { ok: true, branch }; + failures.push(`${branch}: checked out ${currentBranch ?? ''}`); + input.logger.warn( + `[auto-sync] Branch ${branch} for ${input.item.remoteUrl} synced but current branch is ${currentBranch ?? ''}; trying next branch.`, + ); + } catch (err: unknown) { + const message = (err as Error).message; + if (message.includes('timed out')) sawTimeout = true; + failures.push(`${branch}: ${message}`); + input.logger.warn( + `[auto-sync] Branch ${branch} unavailable for ${input.item.remoteUrl}: ${message}`, + ); + } + } + return { + ok: false, + status: sawTimeout ? 'sync_timeout' : 'branch_unavailable', + message: failures.join('; '), + }; +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts index 579b18ab5..8d4b00a16 100644 --- a/gitnexus/src/core/auto-sync/starter.ts +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -1,28 +1,95 @@ -import { loadAutoSyncConfig, parseAutoSyncFlag } from './config.js'; +import fs from 'node:fs/promises'; +import crypto from 'node:crypto'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { loadAutoSyncConfig } from './config.js'; import { runAutoSyncOnce } from './runner.js'; +import { getAutoSyncWatchDir } from './state.js'; export interface AutoSyncStartHandle { - stop(): void; + stop(): Promise; } -export async function maybeStartAutoSyncFromEnv(options: { +export type WatchStatusState = 'running' | 'stopping' | 'stopped' | 'stale' | 'error'; + +export interface WatchStatusRecord { + state: WatchStatusState; + pid?: number; + ownerId?: string; + configPath?: string; + message?: string; + updatedAt: string; +} + +export interface WatchLockRecord { + pid: number; + ownerId: string; + createdAt: string; +} + +export interface AutoSyncWatchPaths { + pidPath: string; + lockPath: string; + statusPath: string; +} + +export interface AutoSyncWatchControlDeps { + isProcessAlive(pid: number): boolean; + killProcess(pid: number, signal?: NodeJS.Signals): void; + sleep(ms: number): Promise; +} + +export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWatchPaths { + const watchDir = getAutoSyncWatchDir(gitnexusDir); + return { + pidPath: path.join(watchDir, 'watch.pid'), + lockPath: path.join(watchDir, 'watch.lock'), + statusPath: path.join(watchDir, 'watch.status.json'), + }; +} + +export async function startAutoSyncWatch(options: { setIntervalFn?: typeof setInterval; clearIntervalFn?: typeof clearInterval; runOnce?: typeof runAutoSyncOnce; stderr?: Pick; + keepAlive?: boolean; + paths?: AutoSyncWatchPaths; + deps?: Partial; } = {}): Promise { const stderr = options.stderr ?? process.stderr; - const flag = parseAutoSyncFlag(); - if (flag.enabled === false) { - if (flag.message) stderr.write(`${flag.message}\n`); - return null; - } + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const ownerId = crypto.randomUUID(); + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + const lockHandle = await acquireWatchLock(paths, deps, stderr); + if (!lockHandle) return null; + await lockHandle.writeFile( + `${JSON.stringify({ pid: process.pid, ownerId, createdAt: new Date().toISOString() })}\n`, + 'utf-8', + ); + await fs.writeFile(paths.pidPath, `${process.pid}\n`, 'utf-8'); const loaded = await loadAutoSyncConfig(); if (loaded.ok === false) { stderr.write(`${loaded.message}\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: process.pid, + ownerId, + message: loaded.message, + updatedAt: new Date().toISOString(), + }); + await cleanupWatchFiles(paths, lockHandle); return null; } + await writeWatchStatus(paths, { + state: 'running', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); const runOnce = options.runOnce ?? runAutoSyncOnce; let running = false; @@ -32,9 +99,17 @@ export async function maybeStartAutoSyncFromEnv(options: { return; } running = true; + const startedAt = new Date(); + stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); void runOnce(loaded.config) + .then((result) => { + stderr.write( + `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, + ); + }) .catch((err: unknown) => { stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + stderr.write('[auto-sync] Watch loop finished: failed.\n'); }) .finally(() => { running = false; @@ -46,6 +121,359 @@ export async function maybeStartAutoSyncFromEnv(options: { const setIntervalFn = options.setIntervalFn ?? setInterval; const clearIntervalFn = options.clearIntervalFn ?? clearInterval; const timer = setIntervalFn(runSafely, intervalMs); - timer.unref?.(); - return { stop: () => clearIntervalFn(timer) }; + if (options.keepAlive === false) timer.unref?.(); + return { + stop: async () => { + clearIntervalFn(timer); + await writeWatchStatus(paths, { + state: 'stopped', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }).finally(() => cleanupWatchFiles(paths, lockHandle)); + }, + }; +} + +async function acquireWatchLock( + paths: AutoSyncWatchPaths, + deps: AutoSyncWatchControlDeps, + stderr: Pick, +): Promise { + try { + return await fs.open(paths.lockPath, 'wx'); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + } + + const lock = await readLockFile(paths.lockPath); + if (!lock) { + const message = 'watch lock already exists but has no readable owner; refusing to start'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return null; + } + + if (deps.isProcessAlive(lock.pid)) { + stderr.write(`[auto-sync] Watch is already running with pid ${lock.pid}.\n`); + await writeWatchStatus(paths, { + state: 'running', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'watch already running', + updatedAt: new Date().toISOString(), + }); + return null; + } + + stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'removed stale lock and pid', + updatedAt: new Date().toISOString(), + }); + + try { + return await fs.open(paths.lockPath, 'wx'); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'EEXIST') { + const message = 'watch lock was reacquired by another process; refusing to start'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return null; + } + throw err; + } +} + +export async function stopAutoSyncWatch(options: { + paths?: AutoSyncWatchPaths; + stderr?: Pick; + deps?: Partial; + timeoutMs?: number; + pollMs?: number; +} = {}): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const timeoutMs = options.timeoutMs ?? 10_000; + const pollMs = options.pollMs ?? 100; + const pid = await readPid(paths.pidPath); + if (!pid) { + const lock = await readLockFile(paths.lockPath); + if (lock && deps.isProcessAlive(lock.pid)) { + const message = `watch appears to be starting with pid ${lock.pid}; pid file is not ready`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: lock.pid, + ownerId: lock.ownerId, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + if (lock) { + stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid: lock.pid, + ownerId: lock.ownerId, + message: 'removed stale lock without pid file', + updatedAt: new Date().toISOString(), + }); + return false; + } + if (await fileExists(paths.lockPath)) { + const message = 'watch lock exists but has no readable owner; refusing to stop'; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + stderr.write('[auto-sync] Watch is not running.\n'); + await writeWatchStatus(paths, { + state: 'stopped', + message: 'no pid file', + updatedAt: new Date().toISOString(), + }); + return false; + } + if (!deps.isProcessAlive(pid)) { + stderr.write(`[auto-sync] Removing stale watch pid ${pid}.\n`); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stale', + pid, + message: 'removed stale pid and lock', + updatedAt: new Date().toISOString(), + }); + return false; + } + const owner = await readVerifiedWatchOwner(paths, pid); + if (owner.ok === false) { + const message = `refusing to stop pid ${pid}; ${owner.reason}`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'error', + pid, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + await writeWatchStatus(paths, { + state: 'stopping', + pid, + ownerId: owner.owner.ownerId, + message: 'stop signal sent; waiting for watch process to exit', + updatedAt: new Date().toISOString(), + }); + deps.killProcess(pid, 'SIGTERM'); + stderr.write(`[auto-sync] Stop signal sent to watch pid ${pid}.\n`); + const stopped = await waitForProcessExit(pid, { deps, timeoutMs, pollMs }); + if (!stopped) { + const message = `watch pid ${pid} did not exit within ${timeoutMs}ms`; + stderr.write(`[auto-sync] ${message}.\n`); + await writeWatchStatus(paths, { + state: 'stopping', + pid, + ownerId: owner.owner.ownerId, + message, + updatedAt: new Date().toISOString(), + }); + return false; + } + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); + await writeWatchStatus(paths, { + state: 'stopped', + pid, + ownerId: owner.owner.ownerId, + message: 'watch stopped', + updatedAt: new Date().toISOString(), + }); + return true; +} + +export async function readAutoSyncWatchStatus( + paths = getAutoSyncWatchPaths(), + deps: Partial = {}, +): Promise { + const resolvedDeps = resolveWatchDeps(deps); + const pid = await readPid(paths.pidPath); + if (pid && !resolvedDeps.isProcessAlive(pid)) { + return { + state: 'stale', + pid, + message: 'pid file exists but process is not running', + updatedAt: new Date().toISOString(), + }; + } + if (pid) { + const stored = await readStatusFile(paths.statusPath); + const owner = await readVerifiedWatchOwner(paths, pid); + if (owner.ok === false) { + return { + ...stored, + state: 'error', + pid, + message: owner.reason, + updatedAt: new Date().toISOString(), + }; + } + return { + ...stored, + state: stored?.state === 'stopping' ? 'stopping' : 'running', + pid, + ownerId: owner.owner.ownerId, + updatedAt: new Date().toISOString(), + }; + } + const stored = await readStatusFile(paths.statusPath); + return stored ?? { state: 'stopped', updatedAt: new Date().toISOString() }; +} + +async function readLockFile(lockPath: string): Promise { + try { + const raw = await fs.readFile(lockPath, 'utf-8'); + const parsed = JSON.parse(raw) as WatchLockRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId + ) { + return parsed; + } + return undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return undefined; + } +} + +async function readVerifiedWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, +): Promise<{ ok: true; owner: WatchLockRecord } | { ok: false; reason: string }> { + const [status, lock] = await Promise.all([ + readStatusFile(paths.statusPath), + readLockFile(paths.lockPath), + ]); + if (!lock) return { ok: false, reason: 'watch lock is missing or invalid' }; + if (!status) return { ok: false, reason: 'watch status is missing or invalid' }; + if (lock.pid !== pid) return { ok: false, reason: 'watch lock pid does not match pid file' }; + if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' }; + if (!status.ownerId || status.ownerId !== lock.ownerId) { + return { ok: false, reason: 'watch status owner does not match lock owner' }; + } + return { ok: true, owner: lock }; +} + +async function waitForProcessExit( + pid: number, + options: { deps: AutoSyncWatchControlDeps; timeoutMs: number; pollMs: number }, +): Promise { + const deadline = Date.now() + options.timeoutMs; + while (Date.now() < deadline) { + if (!options.deps.isProcessAlive(pid)) return true; + await options.deps.sleep(options.pollMs); + } + return !options.deps.isProcessAlive(pid); +} + +async function readPid(pidPath: string): Promise { + try { + const raw = await fs.readFile(pidPath, 'utf-8'); + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 ? pid : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw err; + } +} + +async function readStatusFile(statusPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(statusPath, 'utf-8')) as WatchStatusRecord; + return parsed && typeof parsed === 'object' ? parsed : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return { + state: 'error', + message: `unable to read status file: ${(err as Error).message}`, + updatedAt: new Date().toISOString(), + }; + } +} + +async function writeWatchStatus(paths: AutoSyncWatchPaths, record: WatchStatusRecord): Promise { + await fs.mkdir(path.dirname(paths.statusPath), { recursive: true }); + const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, paths.statusPath); +} + +async function cleanupWatchFiles(paths: AutoSyncWatchPaths, lockHandle?: fs.FileHandle): Promise { + await lockHandle?.close().catch(() => {}); + await removeIfExists(paths.pidPath); + await removeIfExists(paths.lockPath); +} + +async function removeIfExists(filePath: string): Promise { + await fs.rm(filePath, { force: true }); +} + +async function fileExists(filePath: string): Promise { + return fs.access(filePath).then( + () => true, + () => false, + ); +} + +function resolveWatchDeps(deps: Partial = {}): AutoSyncWatchControlDeps { + return { + isProcessAlive: + deps.isProcessAlive ?? + ((pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }), + killProcess: + deps.killProcess ?? + ((pid, signal = 'SIGTERM') => { + process.kill(pid, signal); + }), + sleep: + deps.sleep ?? + ((ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })), + }; } diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts index f14d9aa5f..0de053ca9 100644 --- a/gitnexus/src/core/auto-sync/state.ts +++ b/gitnexus/src/core/auto-sync/state.ts @@ -2,19 +2,29 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { getGlobalDir } from '../../storage/repo-manager.js'; -export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped'; +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped'; export interface AutoSyncCommitStateEntry { codeCommitId: string; analyzedCommitId?: string; lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + analyzeConsecutiveFailures?: number; + lastAnalyzeError?: string; lastSyncTime: string; } export type AutoSyncCommitState = Record; +export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'watch'); +} + export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { - return path.join(gitnexusDir, 'auto-sync-state.json'); + return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json'); +} + +export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt'); } export function buildStateKey(repoPath: string, branch: string): string { @@ -59,3 +69,46 @@ export async function saveAutoSyncState( await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); await fs.rename(tmpPath, statePath); } + +export async function writeProjectCommitInfo( + entries: ProjectCommitInfoEntry[], + infoPath = getProjectCommitInfoPath(), +): Promise { + await fs.mkdir(path.dirname(infoPath), { recursive: true }); + const lines = [ + '# GitNexus auto-sync project commit info', + `updated_at: ${new Date().toISOString()}`, + '', + ...entries.flatMap((entry) => [ + `remote: ${entry.remoteUrl}`, + `local_path: ${entry.localPath}`, + `branch: ${entry.branch ?? ''}`, + `code_commit: ${entry.codeCommitId ?? ''}`, + `analyzed_commit: ${entry.analyzedCommitId ?? ''}`, + `status: ${entry.status}`, + `analyze_consecutive_failures: ${entry.analyzeConsecutiveFailures ?? 0}`, + ...(entry.analyzeFailureThreshold === undefined + ? [] + : [`analyze_failure_threshold: ${entry.analyzeFailureThreshold}`]), + ...(entry.lastAnalyzeError ? [`last_analyze_error: ${entry.lastAnalyzeError}`] : []), + `last_sync_time: ${entry.lastSyncTime}`, + '', + ]), + ]; + const tmpPath = `${infoPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${lines.join('\n')}\n`, 'utf-8'); + await fs.rename(tmpPath, infoPath); +} + +export interface ProjectCommitInfoEntry { + remoteUrl: string; + localPath: string; + branch?: string; + codeCommitId?: string; + analyzedCommitId?: string; + status: AutoSyncAnalyzeStatus | 'sync_failed' | 'branch_skipped' | 'branch_unavailable' | 'sync_timeout'; + analyzeConsecutiveFailures?: number; + analyzeFailureThreshold?: number; + lastAnalyzeError?: string; + lastSyncTime: string; +} diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index bb4cfcef4..93670aa0d 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -34,7 +34,7 @@ import { fileURLToPath } from 'url'; import { JobManager } from './analyze-job.js'; import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js'; import { - extractRepoName, + extractWebRepoName, getCloneDir, cloneOrPull, warnIfInsecureAzureConfig, @@ -1582,7 +1582,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => try { // Clone if URL provided if (repoUrl && !repoLocalPath) { - const repoName = extractRepoName(repoUrl); + const repoName = extractWebRepoName(repoUrl); targetPath = getCloneDir(repoName); jobManager.updateJob(job.id, { diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 588f36ea3..60f0b44d6 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -11,10 +11,12 @@ import fs from 'fs/promises'; import { isIP } from 'net'; import { logger } from '../core/logger.js'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { sanitizeRepoName } from '../storage/git.js'; import { assertDirectoryOwnerAndPermissions, quarantineAutoSyncPartial, } from '../core/auto-sync/path-security.js'; +import { validateAutoSyncRemoteUrl } from '../core/auto-sync/config.js'; /** * Root directory for all cloned repositories. Targets must resolve inside this. @@ -59,6 +61,26 @@ export function extractRepoName(url: string): string { return name; } +/** + * Derive a clone directory name for the web `/api/analyze` boundary. + * + * The API historically accepted Azure DevOps and similar URLs whose repo + * segment contains spaces or other directory-unsafe characters by sanitizing + * the final segment. Keep that compatibility at the web boundary while leaving + * `extractRepoName()` strict for internal/security-sensitive callers. + */ +export function extractWebRepoName(url: string): string { + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const rawName = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; + const safeName = sanitizeRepoName(rawName); + if (!rawName || safeName === 'unknown') { + throw new Error('Could not extract a valid repository name from URL'); + } + return safeName; +} + /** Get the clone target directory for a repo name. */ export function getCloneDir(repoName: string): string { // Re-validate at the boundary even though extractRepoName already checked — @@ -247,9 +269,20 @@ export interface CloneOrPullOptions { allowedCloneRoot?: string; expectedRepoName?: string; quarantineRoot?: string; + allowAutoSyncSsh?: boolean; + timeoutMs?: number; + branch?: string; runGitForTest?: typeof runGit; } +type RunGitOptions = { + token?: string; + url?: string; + timeoutMs?: number; + timeoutKillGraceMs?: number; + spawnForTest?: typeof spawn; +}; + /** * Build the `git clone` argument list for a given URL and target directory. * @@ -319,6 +352,10 @@ export function buildCloneArgs(url: string, targetDir: string): string[] { return ['clone', '--depth', '1', '--', url, targetDir]; } +export function buildBranchCloneArgs(url: string, targetDir: string, branch: string): string[] { + return ['clone', '--depth', '1', '--branch', branch, '--', url, targetDir]; +} + /** * Normalize a git URL into a comparable form. * @@ -486,7 +523,8 @@ export async function cloneOrPull( // Always validate the requested URL — the prior shape only ran this in // the code path where the repo was cloned. Now it runs unconditionally, // preventing SSRF / blocked-host bypasses even when targetDir already exists. - validateGitUrl(url); + if (options?.allowAutoSyncSsh) validateAutoSyncRemoteUrl(url); + else validateGitUrl(url); await fs.mkdir(cloneRoot, { recursive: true }); if (options?.allowedCloneRoot) { await assertDirectoryOwnerAndPermissions(cloneRoot); @@ -511,7 +549,24 @@ export async function cloneOrPull( // whatever remote the dir was originally cloned from. await assertRemoteMatchesRequestedUrl(safeTarget, url); onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); - await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url }); + const runGitImpl = options?.runGitForTest ?? runGit; + if (options?.branch) { + await runGitImpl(['fetch', '--depth', '1', 'origin', options.branch], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + await runGitImpl(['checkout', options.branch], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + } + await runGitImpl(['pull', '--ff-only'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); } else { if (targetExists) { throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); @@ -522,7 +577,14 @@ export async function cloneOrPull( onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); try { const runGitImpl = options?.runGitForTest ?? runGit; - await runGitImpl(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + const cloneArgs = options?.branch + ? buildBranchCloneArgs(url, safeTarget, options.branch) + : buildCloneArgs(url, safeTarget); + await runGitImpl(cloneArgs, undefined, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); await assertPostRealpathContainment(cloneRoot, safeTarget); } catch (err: unknown) { if (options?.quarantineRoot) { @@ -731,10 +793,11 @@ export function buildGitEnv( function runGit( args: string[], cwd?: string, - options?: { token?: string; url?: string }, + options?: RunGitOptions, ): Promise { return new Promise((resolve, reject) => { - const proc = spawn('git', args, { + const spawnGit = options?.spawnForTest ?? spawn; + const proc = spawnGit('git', args, { cwd, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, @@ -742,21 +805,47 @@ function runGit( }); let stderr = ''; + let settled = false; + let timedOut = false; + let killTimer: NodeJS.Timeout | undefined; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + fn(); + }; + const timer = + options?.timeoutMs && options.timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + proc.kill('SIGTERM'); + killTimer = setTimeout(() => { + proc.kill('SIGKILL'); + }, options.timeoutKillGraceMs ?? 1_000); + }, options.timeoutMs) + : undefined; proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk; }); proc.on('close', (code) => { - if (code === 0) resolve(); + if (timedOut) { + finish(() => reject(new Error(`git ${args[0]} timed out after ${options?.timeoutMs}ms`))); + return; + } + if (code === 0) finish(resolve); else { // Log full stderr internally but don't expose it to API callers (SSRF mitigation) if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`); - reject(new Error(`git ${args[0]} failed (exit code ${code})`)); + finish(() => reject(new Error(`git ${args[0]} failed (exit code ${code})`))); } }); proc.on('error', (err) => { - reject(new Error(`Failed to spawn git: ${err.message}`)); + finish(() => reject(new Error(`Failed to spawn git: ${err.message}`))); }); }); } + +export const runGitForTest = runGit; diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts index 59e358c6a..9e8c5b13e 100644 --- a/gitnexus/test/unit/auto-sync-runner.test.ts +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -6,18 +6,25 @@ import { describe, expect, it, vi } from 'vitest'; import { addRepoToGroup, getConfiguredRepoPath, - maybeStartAutoSyncFromEnv, + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + resolveActualConcurrency, runAutoSyncOnce, + startAutoSyncWatch, + stopAutoSyncWatch, } from '../../src/core/auto-sync/index.js'; -import type { AutoSyncConfig, AutoSyncRunDeps } from '../../src/core/auto-sync/index.js'; +import type { AutoSyncConfig, AutoSyncRunDeps, AutoSyncWatchPaths } from '../../src/core/auto-sync/index.js'; const config: AutoSyncConfig = { - configPath: '/tmp/.gitnexus/sync_config.yml', + configPath: '/tmp/.gitnexus/watch_config.yml', syncIntervalMinutes: 10, + repoGitTimeoutMs: 10_000, + maxConcurrency: 1, + analyzeFailureThreshold: 3, projects: [ { localPath: '/tmp/repos', - gitnexusGroup: 'back_end', + groupName: 'back_end', branches: ['master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }, @@ -26,7 +33,7 @@ const config: AutoSyncConfig = { const cloneRoot = { root: '/tmp/repos', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', quarantineRetentionDays: 14, }; @@ -37,6 +44,25 @@ function withCloneRoot(deps: Partial): Partial }; } +async function writeWatchOwner(paths: AutoSyncWatchPaths, pid: number, ownerId = `owner-${pid}`): Promise { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile(paths.pidPath, `${pid}\n`); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid, ownerId, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ + state: 'running', + pid, + ownerId, + updatedAt: '2026-06-30T00:00:00.000Z', + })}\n`, + ); + return ownerId; +} + describe('auto-sync runner', () => { it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { const deps: Partial = withCloneRoot({ @@ -54,8 +80,10 @@ describe('auto-sync runner', () => { }, })), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -72,7 +100,10 @@ describe('auto-sync runner', () => { { allowedCloneRoot: '/tmp/repos', expectedRepoName: 'qts_account', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', }, ); expect(deps.getCurrentBranch).toHaveBeenCalledWith('/tmp/repos/qts_account'); @@ -87,6 +118,70 @@ describe('auto-sync runner', () => { { name: 'qts_account', allowDuplicateName: true }, ); expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + status: 'success', + }), + ]); + }); + + it('syncs a group when a repo is newly added to the group', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('syncs a group after successful re-analysis even when membership already exists', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-3'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 2 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(1); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); }); it('skips analysis when commit id has not changed', async () => { @@ -105,8 +200,10 @@ describe('auto-sync runner', () => { }, })), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => false), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -131,8 +228,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(async () => 'qts_account'), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => false), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); await runAutoSyncOnce(config, { @@ -147,12 +246,121 @@ describe('auto-sync runner', () => { { allowedCloneRoot: '/tmp/repos', expectedRepoName: 'qts_account', - quarantineRoot: '/tmp/.gitnexus/quarantine', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', }, ); }); - it('skips analysis when the checked out branch is not configured', async () => { + it('falls back through configured branches and analyzes the first pullable branch', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_remoteUrl, _targetDir, _progress, options) => { + if (options?.branch === 'missing') throw new Error('remote branch not found'); + return '/tmp/repos/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 1, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + expect.objectContaining({ branch: 'missing' }), + ); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 2, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/qts_account', + undefined, + expect.objectContaining({ branch: 'develop' }), + ); + expect(deps.runFullAnalysis).toHaveBeenCalledWith( + '/tmp/repos/qts_account', + { branch: 'develop', skipAgentsMd: true, skipSkills: true }, + { onProgress: expect.any(Function) }, + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).not.toHaveBeenCalled(); + }); + + it('records branch_unavailable when all configured branches fail', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => { + throw new Error('remote branch not found'); + }), + getCurrentBranch: vi.fn(), + getCurrentCommit: vi.fn(), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + branch: 'missing', + status: 'branch_unavailable', + }), + ]); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledTimes(2); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch develop unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).toHaveBeenCalledTimes(1); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Repository sync failed for git@gitee.com:qts_server/qts_account.git; no configured branch could be pulled: missing: remote branch not found; develop: remote branch not found', + ); + }); + + it('records branch_unavailable when checkout ends on an unexpected branch', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), @@ -162,8 +370,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -171,16 +381,16 @@ describe('auto-sync runner', () => { logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); expect(deps.runFullAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( - '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch develop is not in configured branches: master.', + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is develop; trying next branch.', ); }); - it('skips analysis when the checked out repository is detached', async () => { + it('records branch_unavailable when the checked out repository is detached', async () => { const warnLogger = vi.fn(); const deps: Partial = withCloneRoot({ cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), @@ -190,8 +400,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(config, { @@ -199,16 +411,16 @@ describe('auto-sync runner', () => { logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); expect(deps.getCurrentCommit).not.toHaveBeenCalled(); expect(deps.runFullAnalysis).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).not.toHaveBeenCalled(); expect(warnLogger).toHaveBeenCalledWith( - '[auto-sync] Skip analysis for /tmp/repos/qts_account; current branch is not in configured branches: master.', + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is ; trying next branch.', ); }); - it('isolates repository, analysis, and group sync failures', async () => { + it('isolates repository and analysis failures without syncing groups for failed analysis', async () => { const errorLogger = vi.fn(); const failingConfig: AutoSyncConfig = { ...config, @@ -235,10 +447,10 @@ describe('auto-sync runner', () => { registerRepo: vi.fn(), loadState: vi.fn(async () => ({})), saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), addRepoToGroup: vi.fn(async () => true), - syncGroupByName: vi.fn(async () => { - throw new Error('group sync failed'); - }), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), }); const result = await runAutoSyncOnce(failingConfig, { @@ -247,11 +459,11 @@ describe('auto-sync runner', () => { now: () => new Date('2026-06-30T00:00:00.000Z'), }); - expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 3 }); + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 2 }); expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); expect(deps.registerRepo).not.toHaveBeenCalled(); expect(deps.addRepoToGroup).toHaveBeenCalledWith(failingConfig.projects[0], 'qts_account'); - expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.syncGroupByName).not.toHaveBeenCalled(); expect(deps.saveState).toHaveBeenCalledWith( expect.objectContaining({ '/tmp/repos/qts_account|master': expect.objectContaining({ @@ -266,11 +478,311 @@ describe('auto-sync runner', () => { expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Analysis failed for /tmp/repos/qts_account'), ); + }); + + it('reports group sync failures after successful analysis', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => { + throw new Error('group sync failed'); + }), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); + expect(deps.addRepoToGroup).toHaveBeenCalledWith(config.projects[0], 'qts_account'); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); expect(errorLogger).toHaveBeenCalledWith( expect.stringContaining('Group sync failed for back_end'), ); }); + it('caps actual concurrency by available memory and runs clone/analyze work concurrently', async () => { + const events: string[] = []; + let releaseFirstClone: (() => void) | undefined; + const concurrentConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 4, + projects: [ + { + ...config.projects[0], + groupName: undefined, + remoteUrls: ['git@github.com:owner/one.git', 'git@gitlab.com:owner/two.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + events.push(`clone-start:${remoteUrl}`); + if (remoteUrl.includes('/one.git')) { + await new Promise((resolve) => { + releaseFirstClone = resolve; + setTimeout(resolve, 0); + }); + } else { + releaseFirstClone?.(); + } + events.push(`clone-end:${remoteUrl}`); + return remoteUrl.includes('/one.git') ? '/tmp/repos/one' : '/tmp/repos/two'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn((repoPath) => (repoPath.endsWith('/one') ? 'one-commit' : 'two-commit')), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 4), + }); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await runAutoSyncOnce(concurrentConfig, { deps, logger }); + + expect(result.synced).toBe(2); + expect(logger.info).toHaveBeenCalledWith( + '[auto-sync] Starting sync loop with max_concurrency=2 analyze_failure_threshold=3.', + ); + expect(events.slice(0, 2)).toEqual([ + 'clone-start:git@github.com:owner/one.git', + 'clone-start:git@gitlab.com:owner/two.git', + ]); + expect(deps.registerRepo).toHaveBeenCalledTimes(2); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('rejects duplicate resolved targetDir before clone work starts', async () => { + const duplicateConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 2, + projects: [ + { + ...config.projects[0], + remoteUrls: ['git@github.com:owner/repo.git', 'git@gitlab.com:group/repo.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(duplicateConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).rejects.toThrow('Duplicate auto-sync targetDir'); + + expect(deps.cloneOrPull).not.toHaveBeenCalled(); + expect(deps.saveState).not.toHaveBeenCalled(); + expect(deps.writeCommitInfo).not.toHaveBeenCalled(); + }); + + it('rejects non auto-sync SSH URLs at runner boundary', async () => { + const invalidConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], remoteUrls: ['https://github.com/owner/repo.git'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(invalidConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.failed).toBe(1); + expect(deps.cloneOrPull).not.toHaveBeenCalled(); + }); + + it('increments analyze failure count and writes threshold details', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => { + throw new Error('parser crashed\nwith stack'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 1, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'parser crashed with stack', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'failed', + analyzeConsecutiveFailures: 2, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed with stack', + }), + ]); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Analysis failed for /tmp/repos/qts_account; consecutive failures 2/3: parser crashed with stack', + ); + }); + + it('skips analyze when consecutive failures have reached the threshold', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 1, failed: 0 }); + expect(deps.runFullAnalysis).not.toHaveBeenCalled(); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 3, + lastAnalyzeError: 'parser crashed', + lastAnalyzeStatus: 'threshold_skipped', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'threshold_skipped', + analyzeConsecutiveFailures: 3, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed', + }), + ]); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Skip analysis for /tmp/repos/qts_account; analyze consecutive failures 3/3 reached threshold. Fix the repository or clear auto-sync state before retrying.', + ); + }); + + it('clears prior analyze failure count after a successful analyze', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runFullAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result.analyzed).toBe(1); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 0, + lastAnalyzeError: undefined, + lastAnalyzeStatus: 'success', + }), + }), + ); + }); + + it('resolves actual concurrency from configured value and memory', () => { + expect(resolveActualConcurrency(8, 10)).toBe(5); + expect(resolveActualConcurrency(8, 1)).toBe(1); + expect(resolveActualConcurrency(2, 10)).toBe(2); + }); + it('detects existing groupPath to registryName mappings as already joined', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-group-')); @@ -288,7 +800,7 @@ describe('auto-sync runner', () => { ].join('\n'), ); - await expect(addRepoToGroup({ gitnexusGroup: 'back_end' }, 'qts_account')).resolves.toBe( + await expect(addRepoToGroup({ groupName: 'back_end' }, 'qts_account')).resolves.toBe( false, ); @@ -304,59 +816,54 @@ describe('auto-sync runner', () => { }); describe('auto-sync starter', () => { - it('does not read config or register timers when the flag is disabled', async () => { - const previous = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '0'; - const setIntervalFn = vi.fn() as unknown as typeof setInterval; - - try { - const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn }); - expect(handle).toBeNull(); - expect(setIntervalFn).not.toHaveBeenCalled(); - } finally { - if (previous === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previous; - } - }); - - it('registers a clearable timer when enabled with a valid config', async () => { - const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; + it('registers a clearable timer with a valid fixed config', async () => { const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); const timer = { unref: vi.fn() }; const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; const runOnce = vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); + const stderr = { write: vi.fn() }; try { - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; process.env.GITNEXUS_HOME = tempDir; await fs.writeFile( - path.join(tempDir, 'sync_config.yml'), + path.join(tempDir, 'watch_config.yml'), [ 'sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', - ' gitnexus_group: back_end', + ' group_name: back_end', ' branch: master', ' remote_urls:', ' - git@gitee.com:qts_server/qts_account.git', ].join('\n'), ); - const handle = await maybeStartAutoSyncFromEnv({ setIntervalFn, clearIntervalFn, runOnce }); + const handle = await startAutoSyncWatch({ + setIntervalFn, + clearIntervalFn, + runOnce, + stderr, + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); expect(handle).not.toBeNull(); expect(runOnce).toHaveBeenCalledTimes(1); expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); expect(timer.unref).toHaveBeenCalled(); + await vi.waitFor(() => { + expect(stderr.write).toHaveBeenCalledWith(expect.stringContaining('[auto-sync] Watch loop started at ')); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch loop finished: synced=0 analyzed=0 skipped=0 failed=0.\n', + ); + }); - handle?.stop(); + await handle?.stop(); expect(clearIntervalFn).toHaveBeenCalledWith(timer); } finally { - if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; await fs.rm(tempDir, { recursive: true, force: true }); @@ -364,7 +871,6 @@ describe('auto-sync starter', () => { }); it('skips overlapping scheduled runs while a previous run is active', async () => { - const previousFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; const previousHome = process.env.GITNEXUS_HOME; const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); const timer = { unref: vi.fn() }; @@ -383,21 +889,20 @@ describe('auto-sync starter', () => { ); try { - process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = '1'; process.env.GITNEXUS_HOME = tempDir; await fs.writeFile( - path.join(tempDir, 'sync_config.yml'), + path.join(tempDir, 'watch_config.yml'), [ 'sync_interval_minutes: 5', 'projects:', ' - local_path: /tmp/repos', ' branch: master', ' remote_urls:', - ' - https://example.com/team/repo.git', + ' - git@github.com:team/repo.git', ].join('\n'), ); - await maybeStartAutoSyncFromEnv({ setIntervalFn, runOnce, stderr }); + await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); scheduled?.(); expect(runOnce).toHaveBeenCalledTimes(1); @@ -411,8 +916,334 @@ describe('auto-sync starter', () => { expect(runOnce).toHaveBeenCalledTimes(2); } finally { - if (previousFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = previousFlag; + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses a second running watch for the same GITNEXUS_HOME', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + await writeWatchOwner(paths, 12345); + const handle = await startAutoSyncWatch({ + paths, + stderr, + deps: { isProcessAlive: vi.fn(() => true) }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('removes stale pid and lock before starting watch', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + try { + process.env.GITNEXUS_HOME = tempDir; + await writeWatchOwner(paths, 12345); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + setIntervalFn, + clearIntervalFn, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + + expect(handle).not.toBeNull(); + expect(await fs.readFile(paths.pidPath, 'utf-8')).toBe(`${process.pid}\n`); + await handle?.stop(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a half-initialized lock when pid has not been written yet', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + stderr, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + deps: { isProcessAlive: vi.fn(() => true) }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith('[auto-sync] Watch is already running with pid 12345.\n'); + expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a live half-initialized lock when stop runs before pid is written', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + const killProcess = vi.fn(); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr, + deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + expect(killProcess).not.toHaveBeenCalled(); + expect(await fs.readFile(paths.lockPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); + expect(status).toMatchObject({ + state: 'error', + pid: 12345, + ownerId: 'starting-owner', + message: expect.stringContaining('appears to be starting'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('removes a stale half-initialized lock when stop runs before pid is written', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'stale-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { isProcessAlive: vi.fn(() => false), killProcess: vi.fn(), sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + const status = JSON.parse(await fs.readFile(paths.statusPath, 'utf-8')); + expect(status).toMatchObject({ + state: 'stale', + pid: 12345, + ownerId: 'stale-owner', + message: 'removed stale lock without pid file', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports status and sends stop signals from pid files', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + let alive = true; + try { + await writeWatchOwner(paths, 12345); + + await expect( + readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) }), + ).resolves.toMatchObject({ state: 'running', pid: 12345 }); + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + pollMs: 1, + deps: { + isProcessAlive: vi.fn(() => alive), + killProcess: vi.fn((pid, signal) => { + killProcess(pid, signal); + alive = false; + }), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(true); + + expect(killProcess).toHaveBeenCalledWith(12345, 'SIGTERM'); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + await expect(readAutoSyncWatchStatus(paths)).resolves.toMatchObject({ state: 'stopped' }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not mark stopped when stop times out waiting for the owner process', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + timeoutMs: 2, + pollMs: 1, + deps: { + isProcessAlive: vi.fn(() => true), + killProcess: vi.fn(), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(false); + + await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + state: 'stopping', + pid: 12345, + message: expect.stringContaining('did not exit'), + }); + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses to stop when pid status and lock ownership disagree', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const killProcess = vi.fn(); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile(paths.pidPath, '12345\n'); + await fs.writeFile( + paths.lockPath, + `${JSON.stringify({ pid: 12345, ownerId: 'lock-owner', createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ state: 'running', pid: 12345, ownerId: 'other-owner', updatedAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { isProcessAlive: vi.fn(() => true), killProcess, sleep: vi.fn(async () => {}) }, + }), + ).resolves.toBe(false); + + expect(killProcess).not.toHaveBeenCalled(); + await expect(readAutoSyncWatchStatus(paths, { isProcessAlive: vi.fn(() => true) })).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('owner'), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('restart can start only after stop confirms pid and lock cleanup', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + let alive = true; + try { + process.env.GITNEXUS_HOME = tempDir; + await writeWatchOwner(paths, 12345); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + await expect( + stopAutoSyncWatch({ + paths, + timeoutMs: 10, + pollMs: 1, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => alive), + killProcess: vi.fn(() => { + alive = false; + }), + sleep: vi.fn(async () => {}), + }, + }), + ).resolves.toBe(true); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.lockPath)).rejects.toThrow(); + + const handle = await startAutoSyncWatch({ + paths, + setIntervalFn, + clearIntervalFn, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + expect(handle).not.toBeNull(); + await handle?.stop(); + } finally { if (previousHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = previousHome; await fs.rm(tempDir, { recursive: true, force: true }); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts index 6cc636ba6..cc8be6e17 100644 --- a/gitnexus/test/unit/auto-sync.test.ts +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -5,20 +5,25 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { extractRepoNameFromRemoteUrl, + getAutoSyncStatePath, + getAutoSyncWatchDir, + getProjectCommitInfoPath, loadAutoSyncConfig, - parseAutoSyncFlag, parseBranchCandidates, + parseDurationMs, resolveConfiguredCloneRoot, loadAutoSyncState, saveAutoSyncState, shouldAnalyzeCommit, + validateAutoSyncRemoteUrl, + validateAutoSyncBranchName, + writeProjectCommitInfo, } from '../../src/core/auto-sync/index.js'; describe('auto-sync', () => { let tempDir: string; let gitnexusHome: string; let oldHome: string | undefined; - let oldFlag: string | undefined; beforeEach(async () => { const base = path.join(process.cwd(), '.tmp-test'); @@ -27,43 +32,38 @@ describe('auto-sync', () => { gitnexusHome = path.join(tempDir, '.gitnexus'); await fs.mkdir(gitnexusHome); oldHome = process.env.GITNEXUS_HOME; - oldFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; process.env.GITNEXUS_HOME = gitnexusHome; - delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; }); afterEach(async () => { if (oldHome === undefined) delete process.env.GITNEXUS_HOME; else process.env.GITNEXUS_HOME = oldHome; - if (oldFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG; - else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = oldFlag; await fs.rm(tempDir, { recursive: true, force: true }); vi.restoreAllMocks(); }); - it('keeps auto sync disabled when the flag is unset or 0', () => { - expect(parseAutoSyncFlag(undefined)).toEqual({ enabled: false, reason: 'unset' }); - expect(parseAutoSyncFlag('0')).toEqual({ enabled: false, reason: 'disabled' }); + it('places watch runtime artifacts under the watch directory by default', () => { + expect(getAutoSyncWatchDir(gitnexusHome)).toBe(path.join(gitnexusHome, 'watch')); + expect(getAutoSyncStatePath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'auto-sync-state.json'), + ); + expect(getProjectCommitInfoPath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'project_commit_info.txt'), + ); }); - it('enables auto sync only for the explicit value 1', () => { - expect(parseAutoSyncFlag('1')).toEqual({ enabled: true }); - expect(parseAutoSyncFlag('true')).toEqual({ - enabled: false, - reason: 'invalid', - message: '[auto-sync] AUTO_UPDATE_AND_ANALYZE_FLAG must be 0 or 1; got "true". Auto sync is disabled.', - }); - }); - - it('loads sync_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { + it('loads watch_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { await fs.writeFile( - path.join(gitnexusHome, 'sync_config.yml'), + path.join(gitnexusHome, 'watch_config.yml'), [ 'sync_interval_minutes: 10', + 'max_concurrency: 3', + 'repo_git_timeout: 12s', + 'analyze_failure_threshold: 2', 'projects:', ' - local_path: /tmp/repos', - ' gitnexus_group: back_end', - ' branch: test, master, test', + ' group_name: back_end', + ' branches: [test, master, test]', ' remote_urls:', ' - git@gitee.com:qts_server/qts_account.git', ].join('\n'), @@ -73,49 +73,97 @@ describe('auto-sync', () => { expect(loaded.ok).toBe(true); if (!loaded.ok) throw new Error('expected config to load'); - expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'sync_config.yml')); + expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'watch_config.yml')); expect(loaded.config.syncIntervalMinutes).toBe(10); + expect(loaded.config.maxConcurrency).toBe(3); + expect(loaded.config.repoGitTimeoutMs).toBe(12_000); + expect(loaded.config.analyzeFailureThreshold).toBe(2); expect(loaded.config.projects[0]).toMatchObject({ localPath: '/tmp/repos', - gitnexusGroup: 'back_end', + groupName: 'back_end', branches: ['test', 'master'], remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], }); }); + it('defaults repo_git_timeout and max_concurrency and allows empty group_name', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' group_name: ""', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config'); + expect(loaded.config.repoGitTimeoutMs).toBe(10_000); + expect(loaded.config.maxConcurrency).toBe(1); + expect(loaded.config.analyzeFailureThreshold).toBe(3); + expect(loaded.config.projects[0].groupName).toBeUndefined(); + }); + + it('rejects invalid analyze_failure_threshold values', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'analyze_failure_threshold: 1', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('analyze_failure_threshold must be an integer >= 2'); + }); + it('reports missing config without throwing', async () => { const loaded = await loadAutoSyncConfig(); expect(loaded).toEqual({ ok: false, reason: 'missing', - message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'sync_config.yml')}. Auto sync is skipped.`, + message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'watch_config.yml')}. Auto sync is skipped.`, }); }); it('reports invalid config without throwing', async () => { - await fs.writeFile(path.join(gitnexusHome, 'sync_config.yml'), 'projects: []\n'); + await fs.writeFile(path.join(gitnexusHome, 'watch_config.yml'), 'projects: []\n'); const loaded = await loadAutoSyncConfig(); expect(loaded.ok).toBe(false); if (loaded.ok) throw new Error('expected invalid config'); expect(loaded.reason).toBe('invalid'); - expect(loaded.message).toContain('[auto-sync] Invalid sync_config.yml:'); + expect(loaded.message).toContain('[auto-sync] Invalid watch_config.yml:'); expect(loaded.message).toContain('sync_interval_minutes must be a positive integer'); expect(loaded.message).toContain('projects must contain at least one project'); }); it('rejects missing, relative, and traversal local_path values at config load', async () => { await fs.writeFile( - path.join(gitnexusHome, 'sync_config.yml'), + path.join(gitnexusHome, 'watch_config.yml'), [ 'sync_interval_minutes: 10', 'projects:', ' - local_path: ../repos', ' branch: master', ' remote_urls:', - ' - https://example.com/team/repo.git', + ' - git@github.com:team/repo.git', ].join('\n'), ); @@ -147,6 +195,18 @@ describe('auto-sync', () => { } }); + it('allows the default GitNexus repos directory as an auto-sync clone root', async () => { + const root = path.join(gitnexusHome, 'repos'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + }); + it('rejects symlinks in configured clone root paths', async () => { const realRoot = path.join(tempDir, 'real-root'); const linkRoot = path.join(tempDir, 'link-root'); @@ -163,33 +223,88 @@ describe('auto-sync', () => { await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( expect.objectContaining({ root, - quarantineRoot: path.join(gitnexusHome, 'quarantine'), + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), quarantineRetentionDays: 14, }), ); }); + it('creates missing configured clone roots before watch clone work', async () => { + const root = path.join(tempDir, 'missing-repos'); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + expect((await fs.stat(root)).isDirectory()).toBe(true); + }); + it('parses branch strings and arrays with trimming and de-duplication', () => { expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']); expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']); }); + it('rejects unsafe auto-sync branch names', () => { + expect(() => validateAutoSyncBranchName('feature/good-branch')).not.toThrow(); + expect(() => validateAutoSyncBranchName('-upload-pack=evil')).toThrow('must not start'); + expect(() => validateAutoSyncBranchName('feature bad')).toThrow('whitespace'); + expect(() => validateAutoSyncBranchName('feature..bad')).toThrow('must not contain ".."'); + expect(() => validateAutoSyncBranchName('bad:ref')).toThrow('not allowed'); + }); + it('extracts safe repository names from remote URLs', () => { expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( 'qts_account', ); - expect(extractRepoNameFromRemoteUrl('https://example.com/team/repo-name.git')).toBe( - 'repo-name', - ); + expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe('repo-name'); }); it('rejects unsafe repository names without sanitizing them', () => { - expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/repo$name.git')).toThrow( + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/repo$name.git')).toThrow( 'valid repository name', ); - expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/..')).toThrow( - 'valid repository name', + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/..')).toThrow('traversal'); + }); + + it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => { + expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('user@github.com:owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow('host must be'); + }); + + it('parses repo git timeout durations', () => { + expect(parseDurationMs('10s')).toBe(10_000); + expect(parseDurationMs('2m')).toBe(120_000); + expect(parseDurationMs('5000ms')).toBe(5000); + expect(parseDurationMs('10')).toBe(10_000); + expect(parseDurationMs(10)).toBe(10_000); + }); + + it('keeps branch compatibility but rejects branch and branches together', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' branches: [develop]', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('must not set both branch and branches'); }); it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => { @@ -217,6 +332,8 @@ describe('auto-sync', () => { codeCommitId: 'abc', analyzedCommitId: 'abc', lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', lastSyncTime: '2026-06-30T00:00:00.000Z', }, }, @@ -228,11 +345,13 @@ describe('auto-sync', () => { ); await expect(loadAutoSyncState(statePath)).resolves.toEqual({ '/tmp/repos/qts_account|master': { - codeCommitId: 'abc', - analyzedCommitId: 'abc', - lastAnalyzeStatus: 'success', - lastSyncTime: '2026-06-30T00:00:00.000Z', - }, + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, }); }); @@ -247,4 +366,48 @@ describe('auto-sync', () => { `[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`, ); }); + + it('writes project_commit_info.txt atomically', async () => { + const infoPath = path.join(tempDir, 'project_commit_info.txt'); + + await writeProjectCommitInfo( + [ + { + remoteUrl: 'git@github.com:owner/repo.git', + localPath: '/tmp/repos/repo', + branch: 'master', + codeCommitId: 'abc', + analyzedCommitId: 'abc', + status: 'success', + analyzeConsecutiveFailures: 0, + analyzeFailureThreshold: 3, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + { + remoteUrl: 'git@github.com:owner/bad.git', + localPath: '/tmp/repos/bad', + branch: 'master', + codeCommitId: 'def', + analyzedCommitId: 'abc', + status: 'threshold_skipped', + analyzeConsecutiveFailures: 3, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + ], + infoPath, + ); + + const content = await fs.readFile(infoPath, 'utf-8'); + expect(content).toContain('remote: git@github.com:owner/repo.git'); + expect(content).toContain('code_commit: abc'); + expect(content).toContain('analyze_consecutive_failures: 0'); + expect(content).toContain('analyze_failure_threshold: 3'); + expect(content).toContain('status: threshold_skipped'); + expect(content).toContain('last_analyze_error: parser crashed'); + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('project_commit_info.txt.tmp'), + ); + }); }); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 8bd6fcee9..6eb9b09ab 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -17,7 +17,11 @@ function runHelp(command: string, env: NodeJS.ProcessEnv = {}) { } function runHelpArgs(args: string[], env: NodeJS.ProcessEnv = {}) { - return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args, '--help'], { + return runCliArgs([...args, '--help'], env); +} + +function runCliArgs(args: string[], env: NodeJS.ProcessEnv = {}) { + return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args], { cwd: repoRoot, encoding: 'utf8', env: { ...process.env, ...env }, @@ -236,6 +240,42 @@ describe('CLI help surface', () => { } }); + it('watch help exposes lifecycle actions and state files', () => { + const result = runHelp('watch'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('gitnexus watch [options] [action]'); + expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch_config.yml'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/watch.pid'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/project_commit_info.txt'); + }); + + it('watch init creates the default watch_config.yml and does not overwrite it', () => { + const home = fs.mkdtempSync(path.join(repoRoot, '.tmp-test/gitnexus-watch-init-')); + try { + const first = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); + const configPath = path.join(home, 'watch_config.yml'); + + expect(first.status).toBe(0); + expect(first.stdout).toContain(`Created ${configPath}`); + const config = fs.readFileSync(configPath, 'utf8'); + expect(config).toContain('sync_interval_minutes: 10'); + expect(config).toContain('analyze_failure_threshold: 3'); + expect(config).toContain(`local_path: ${path.join(home, 'repo')}`); + expect(config).not.toContain('/abs/path/to/repos'); + expect(config).toContain('git@github.com:owner/repo.git'); + + const second = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home }); + + expect(second.status).toBe(1); + expect(second.stderr).toContain(`Config already exists: ${configPath}`); + expect(fs.readFileSync(configPath, 'utf8')).toBe(config); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it('wiki help shows provider, review, and verbose flags', () => { const result = runHelp('wiki'); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 9d879a22a..c16c7eb2a 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -16,20 +16,24 @@ vi.mock('../../src/core/logger.js', () => ({ import { extractRepoName, + extractWebRepoName, getCloneDir, validateGitUrl, cloneOrPull, buildCloneArgs, + buildBranchCloneArgs, buildGitEnv, normalizeGitUrlForCompare, assertRemoteMatchesRequestedUrl, isAzureDevOpsUrl, warnIfInsecureAzureConfig, + runGitForTest, } from '../../src/server/git-clone.js'; import path from 'node:path'; import os from 'node:os'; import fs from 'node:fs/promises'; import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; import { getRemoteOriginUrl } from '../../src/storage/git.js'; import { getGlobalDir } from '../../src/storage/repo-manager.js'; @@ -365,6 +369,20 @@ describe('git-clone', () => { expect(args.some((a) => a.toLowerCase().includes('authorization'))).toBe(false); expect(args.some((a) => a.includes('extraHeader'))).toBe(false); }); + + it('adds --branch before the URL separator for branch-specific clones', () => { + const args = buildBranchCloneArgs('git@github.com:owner/repo.git', '/safe/target', 'develop'); + expect(args).toEqual([ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@github.com:owner/repo.git', + '/safe/target', + ]); + }); }); describe('buildGitEnv — token injection', () => { @@ -551,6 +569,57 @@ describe('git-clone', () => { ); }); + it('keeps regular cloneOrPull restricted to http and https URLs', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('git@github.com:owner/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('Invalid URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('allows auto-sync SSH SCP clone URLs with a per-repo timeout', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const runGitForTest = vi.fn(async () => { + await fs.mkdir(target); + }); + try { + await expect( + cloneOrPull('git@gitlab.com:group/subgroup/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'develop', + runGitForTest, + }), + ).resolves.toBe(target); + + expect(runGitForTest).toHaveBeenCalledWith( + [ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@gitlab.com:group/subgroup/repo.git', + target, + ], + undefined, + { token: undefined, url: 'git@gitlab.com:group/subgroup/repo.git', timeoutMs: 10_000 }, + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('allows an explicitly controlled auto-sync clone root outside the default root', async () => { const root = await mkControlledRoot('gitnexus-controlled-root-'); try { @@ -810,6 +879,30 @@ describe('git-clone', () => { }); }); + describe('extractWebRepoName — API clone compatibility', () => { + it('sanitizes repo names with spaces and unsafe directory characters at the web boundary', () => { + expect( + extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces'), + ).toBe('My_Repo_With_Spaces'); + expect(extractWebRepoName('https://example.com/team/repo$name.git')).toBe('repo_name'); + }); + + it('keeps Windows reserved names from becoming clone directories', () => { + expect(() => extractWebRepoName('https://example.com/team/CON.git')).toThrow( + 'valid repository name', + ); + expect(() => extractWebRepoName('https://example.com/team/NUL.txt')).toThrow( + 'valid repository name', + ); + }); + + it('leaves strict extractRepoName behavior unchanged for internal callers', () => { + expect(() => extractRepoName('https://example.com/team/repo$name.git')).toThrow( + 'valid repository name', + ); + }); + }); + describe('validateGitUrl — Azure DevOps URLs', () => { it('allows self-hosted Azure DevOps Server URLs', () => { expect(() => @@ -988,4 +1081,41 @@ describe('git-clone', () => { } }); }); + + describe('runGit timeout', () => { + it('waits for close and sends SIGKILL after the grace period before returning timeout', async () => { + vi.useFakeTimers(); + try { + const child = new EventEmitter() as EventEmitter & { + stderr: EventEmitter; + kill: ReturnType; + }; + child.stderr = new EventEmitter(); + child.kill = vi.fn(); + const spawnForTest = vi.fn(() => child) as unknown as typeof spawn; + + const promise = runGitForTest(['clone'], undefined, { + timeoutMs: 20, + timeoutKillGraceMs: 20, + spawnForTest, + }); + + await vi.advanceTimersByTimeAsync(25); + let settled = false; + promise.catch(() => {}).finally(() => { + settled = true; + }); + await vi.runAllTicks(); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(25); + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + child.emit('close', null); + await expect(promise).rejects.toThrow('timed out after 20ms'); + } finally { + vi.useRealTimers(); + } + }); + }); });