mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-09 03:17:54 +00:00
adds an opt-in auto sync and analysis loop for GitNexus,gitnexus watch [init|start|restart|stop|status]
This commit is contained in:
weiyf
parent
6d6b359639
commit
fbde7b3e02
19 changed files with 2336 additions and 230 deletions
22
README.md
22
README.md
|
|
@ -240,6 +240,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl
|
|||
gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses
|
||||
gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB)
|
||||
gitnexus analyze --workers <n> # Parse worker pool size (>=1; default: cores-1, capped at 16, auto-sized to the repo). 0 is rejected — there is no sequential mode.
|
||||
gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml
|
||||
gitnexus mcp # Start MCP server (stdio) — serves all indexed repos
|
||||
gitnexus serve # Start local HTTP server (multi-repo) for web UI connection
|
||||
gitnexus list # List all indexed repositories
|
||||
|
|
@ -262,6 +263,27 @@ gitnexus group query <name> <q> # Search execution flows across all repos in a
|
|||
gitnexus group status <name> # Check staleness of repos in a group
|
||||
```
|
||||
|
||||
### `gitnexus watch`
|
||||
|
||||
`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output.
|
||||
|
||||
```yaml
|
||||
sync_interval_minutes: 10
|
||||
max_concurrency: 1
|
||||
repo_git_timeout: 10s
|
||||
analyze_failure_threshold: 3
|
||||
projects:
|
||||
- local_path: /abs/path/to/repos
|
||||
branches: [master, main]
|
||||
group_name: back_end
|
||||
remote_urls:
|
||||
- git@github.com:owner/repo.git
|
||||
- git@gitlab.com:group/repo.git
|
||||
- git@gitee.com:owner/repo.git
|
||||
```
|
||||
|
||||
`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped by `floor(availableMemoryGB / 2)` with a minimum of `1`, printed at each loop start. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group processing.
|
||||
|
||||
> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove.
|
||||
|
||||
If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget.
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl
|
|||
gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768)
|
||||
gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses
|
||||
gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB)
|
||||
gitnexus watch [init|start|restart|stop|status] # Control auto-sync from GITNEXUS_HOME/watch_config.yml
|
||||
gitnexus mcp # Start MCP server (stdio) — serves all indexed repos
|
||||
gitnexus serve # Start local HTTP server (multi-repo) for web UI
|
||||
gitnexus index # Register an existing .gitnexus/ folder into the global registry
|
||||
|
|
@ -197,6 +198,27 @@ gitnexus group query <name> <q> # Search execution flows across all repos in a
|
|||
gitnexus group status <name> # Check staleness of repos in a group
|
||||
```
|
||||
|
||||
### `gitnexus watch`
|
||||
|
||||
`gitnexus watch` is the explicit long-running auto-sync entrypoint. `gitnexus watch init` creates a default `GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus watch` is the same as `gitnexus watch start`; `restart`, `stop`, and `status` manage the same `GITNEXUS_HOME` instance. It reads only `GITNEXUS_HOME/watch_config.yml`, runs once immediately, then repeats on `sync_interval_minutes`. Watch runtime artifacts live under `GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.pid`, `watch.lock`, and `watch.status.json` prevent multiple watch processes for one home, and `quarantine/` stores partial clone output.
|
||||
|
||||
```yaml
|
||||
sync_interval_minutes: 10
|
||||
max_concurrency: 1
|
||||
repo_git_timeout: 10s
|
||||
analyze_failure_threshold: 3
|
||||
projects:
|
||||
- local_path: /abs/path/to/repos
|
||||
branches: [master, main]
|
||||
group_name: back_end
|
||||
remote_urls:
|
||||
- git@github.com:owner/repo.git
|
||||
- git@gitlab.com:group/repo.git
|
||||
- git@gitee.com:owner/repo.git
|
||||
```
|
||||
|
||||
`remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and skips repeated failing analyze runs for the same repo branch until the auto-sync state is cleared. Use `branches` to try branches in order; legacy `branch` remains supported. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project. `GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `GITNEXUS_HOME/watch/auto-sync-state.json`.
|
||||
|
||||
> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove.
|
||||
|
||||
## Remote Embeddings
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const COMMAND_DESCRIPTION_KEYS = {
|
|||
'': 'help.description.root',
|
||||
setup: 'help.command.setup.description',
|
||||
uninstall: 'help.command.uninstall.description',
|
||||
watch: 'help.command.watch.description',
|
||||
analyze: 'help.command.analyze.description',
|
||||
index: 'help.command.index.description',
|
||||
serve: 'help.command.serve.description',
|
||||
|
|
|
|||
|
|
@ -120,6 +120,8 @@ export const en = {
|
|||
'One-time setup: configure MCP for Cursor, Claude Code, OpenCode, Codex',
|
||||
'help.command.uninstall.description':
|
||||
'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors',
|
||||
'help.command.watch.description':
|
||||
'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml',
|
||||
'help.command.analyze.description': 'Index a repository (full analysis)',
|
||||
'help.command.index.description':
|
||||
'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)',
|
||||
|
|
|
|||
|
|
@ -122,6 +122,8 @@ export const zhCN = {
|
|||
'help.command.setup.description': '一次性设置：为 Cursor、Claude Code、OpenCode、Codex 配置 MCP',
|
||||
'help.command.uninstall.description':
|
||||
'撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子',
|
||||
'help.command.watch.description':
|
||||
'控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析',
|
||||
'help.command.analyze.description': '索引仓库（完整分析）',
|
||||
'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）',
|
||||
'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器',
|
||||
|
|
|
|||
|
|
@ -15,12 +15,6 @@ const _require = createRequire(import.meta.url);
|
|||
const pkg = _require('../../package.json');
|
||||
const program = new Command();
|
||||
|
||||
if (process.env.AUTO_UPDATE_AND_ANALYZE_FLAG?.trim() === '1') {
|
||||
void import('../core/auto-sync/index.js').then(({ maybeStartAutoSyncFromEnv }) =>
|
||||
maybeStartAutoSyncFromEnv(),
|
||||
);
|
||||
}
|
||||
|
||||
function collectCodingAgents(value: string, previous: string[] | undefined): string[] {
|
||||
return [...(previous ?? []), ...value.split(',')];
|
||||
}
|
||||
|
|
@ -47,6 +41,25 @@ program
|
|||
.option('-f, --force', 'Apply the changes (default is a dry-run preview)')
|
||||
.action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand'));
|
||||
|
||||
program
|
||||
.command('watch [action]')
|
||||
.description(
|
||||
'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml',
|
||||
)
|
||||
.addHelpText(
|
||||
'after',
|
||||
[
|
||||
'',
|
||||
'Actions: init, start (default), restart, stop, status',
|
||||
'Configuration: GITNEXUS_HOME/watch_config.yml',
|
||||
'Runtime files: GITNEXUS_HOME/watch/watch.pid, watch.lock, watch.status.json, auto-sync-state.json',
|
||||
'Writes: GITNEXUS_HOME/watch/project_commit_info.txt',
|
||||
'Remote URLs: only git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, and git@gitee.com:owner/repo.git are allowed.',
|
||||
'Runs once immediately, then repeats on sync_interval_minutes.',
|
||||
].join('\n'),
|
||||
)
|
||||
.action(createLazyAction(() => import('./watch.js'), 'watchCommand'));
|
||||
|
||||
// Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook
|
||||
// before it overwrites the var, so the postAction hook can restore it. The
|
||||
// analyzeCommand env snapshot is taken AFTER this hook runs, so it cannot undo
|
||||
|
|
|
|||
|
|
@ -7,14 +7,18 @@ import { normalizeConfiguredCloneRoot } from './path-security.js';
|
|||
const _require = createRequire(import.meta.url);
|
||||
const yaml = _require('js-yaml') as typeof import('js-yaml');
|
||||
|
||||
export const AUTO_SYNC_FLAG = 'AUTO_UPDATE_AND_ANALYZE_FLAG';
|
||||
export const AUTO_SYNC_CONFIG_FILE = 'sync_config.yml';
|
||||
export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml';
|
||||
const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
|
||||
const MIN_SYNC_INTERVAL_MINUTES = 5;
|
||||
const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_MAX_CONCURRENCY = 1;
|
||||
export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3;
|
||||
const MIN_ANALYZE_FAILURE_THRESHOLD = 2;
|
||||
const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']);
|
||||
|
||||
export interface AutoSyncProjectConfig {
|
||||
localPath: string;
|
||||
gitnexusGroup?: string;
|
||||
groupName?: string;
|
||||
branches: string[];
|
||||
remoteUrls: string[];
|
||||
}
|
||||
|
|
@ -22,29 +26,16 @@ export interface AutoSyncProjectConfig {
|
|||
export interface AutoSyncConfig {
|
||||
configPath: string;
|
||||
syncIntervalMinutes: number;
|
||||
repoGitTimeoutMs: number;
|
||||
maxConcurrency: number;
|
||||
analyzeFailureThreshold: number;
|
||||
projects: AutoSyncProjectConfig[];
|
||||
}
|
||||
|
||||
export type AutoSyncFlagDecision =
|
||||
| { enabled: true }
|
||||
| { enabled: false; reason: 'unset' | 'disabled' | 'invalid'; message?: string };
|
||||
|
||||
export type AutoSyncConfigLoadResult =
|
||||
| { ok: true; config: AutoSyncConfig }
|
||||
| { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string };
|
||||
|
||||
export function parseAutoSyncFlag(raw = process.env[AUTO_SYNC_FLAG]): AutoSyncFlagDecision {
|
||||
if (raw === undefined || raw.trim() === '') return { enabled: false, reason: 'unset' };
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed === '0') return { enabled: false, reason: 'disabled' };
|
||||
if (trimmed === '1') return { enabled: true };
|
||||
return {
|
||||
enabled: false,
|
||||
reason: 'invalid',
|
||||
message: `[auto-sync] ${AUTO_SYNC_FLAG} must be 0 or 1; got "${trimmed}". Auto sync is disabled.`,
|
||||
};
|
||||
}
|
||||
|
||||
export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string {
|
||||
return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE);
|
||||
}
|
||||
|
|
@ -92,7 +83,7 @@ export async function loadAutoSyncConfig(
|
|||
return {
|
||||
ok: false,
|
||||
reason: 'invalid',
|
||||
message: `[auto-sync] Invalid sync_config.yml: ${(err as Error).message}. Auto sync is skipped.`,
|
||||
message: `[auto-sync] Invalid watch_config.yml: ${(err as Error).message}. Auto sync is skipped.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -111,6 +102,31 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy
|
|||
errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`);
|
||||
}
|
||||
|
||||
const maxConcurrency =
|
||||
raw.max_concurrency === undefined ? DEFAULT_MAX_CONCURRENCY : Number(raw.max_concurrency);
|
||||
if (!Number.isInteger(maxConcurrency) || maxConcurrency <= 0) {
|
||||
errors.push('max_concurrency must be a positive integer');
|
||||
}
|
||||
|
||||
const repoGitTimeoutMs =
|
||||
raw.repo_git_timeout === undefined
|
||||
? DEFAULT_REPO_GIT_TIMEOUT_MS
|
||||
: parseDurationMs(raw.repo_git_timeout);
|
||||
if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) {
|
||||
errors.push('repo_git_timeout must be a positive duration such as 10s');
|
||||
}
|
||||
|
||||
const analyzeFailureThreshold =
|
||||
raw.analyze_failure_threshold === undefined
|
||||
? DEFAULT_ANALYZE_FAILURE_THRESHOLD
|
||||
: Number(raw.analyze_failure_threshold);
|
||||
if (
|
||||
!Number.isInteger(analyzeFailureThreshold) ||
|
||||
analyzeFailureThreshold < MIN_ANALYZE_FAILURE_THRESHOLD
|
||||
) {
|
||||
errors.push(`analyze_failure_threshold must be an integer >= ${MIN_ANALYZE_FAILURE_THRESHOLD}`);
|
||||
}
|
||||
|
||||
const rawProjects = raw.projects;
|
||||
if (!Array.isArray(rawProjects) || rawProjects.length === 0) {
|
||||
errors.push('projects must contain at least one project');
|
||||
|
|
@ -142,22 +158,86 @@ export function parseAutoSyncConfig(content: string, configPath: string): AutoSy
|
|||
if (remoteUrls.length === 0) {
|
||||
errors.push(`projects[${index}].remote_urls must contain at least one URL`);
|
||||
}
|
||||
for (let urlIndex = 0; urlIndex < remoteUrls.length; urlIndex += 1) {
|
||||
try {
|
||||
validateAutoSyncRemoteUrl(remoteUrls[urlIndex]);
|
||||
} catch (err: unknown) {
|
||||
errors.push(`projects[${index}].remote_urls[${urlIndex}] ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const branches = parseBranchCandidates(project.branch);
|
||||
if (branches.length === 0) errors.push(`projects[${index}].branch is required`);
|
||||
if (project.branch !== undefined && project.branches !== undefined) {
|
||||
errors.push(`projects[${index}] must not set both branch and branches`);
|
||||
}
|
||||
const branches = parseBranchCandidates(
|
||||
project.branches !== undefined ? project.branches : project.branch,
|
||||
);
|
||||
if (branches.length === 0) errors.push(`projects[${index}].branches is required`);
|
||||
for (let branchIndex = 0; branchIndex < branches.length; branchIndex += 1) {
|
||||
try {
|
||||
validateAutoSyncBranchName(branches[branchIndex]);
|
||||
} catch (err: unknown) {
|
||||
errors.push(`projects[${index}].branches[${branchIndex}] ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const gitnexusGroup =
|
||||
typeof project.gitnexus_group === 'string' ? project.gitnexus_group.trim() : undefined;
|
||||
if (gitnexusGroup && !GROUP_NAME_PATTERN.test(gitnexusGroup)) {
|
||||
errors.push(`projects[${index}].gitnexus_group is invalid`);
|
||||
const groupName =
|
||||
typeof project.group_name === 'string' && project.group_name.trim()
|
||||
? project.group_name.trim()
|
||||
: undefined;
|
||||
if (groupName && !GROUP_NAME_PATTERN.test(groupName)) {
|
||||
errors.push(`projects[${index}].group_name is invalid`);
|
||||
}
|
||||
|
||||
if (localPath && remoteUrls.length > 0 && branches.length > 0) {
|
||||
projects.push({ localPath, gitnexusGroup, branches, remoteUrls });
|
||||
projects.push({ localPath, groupName, branches, remoteUrls });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (errors.length > 0) throw new Error(errors.join('; '));
|
||||
return { configPath, syncIntervalMinutes: interval, projects };
|
||||
return {
|
||||
configPath,
|
||||
syncIntervalMinutes: interval,
|
||||
repoGitTimeoutMs,
|
||||
maxConcurrency,
|
||||
analyzeFailureThreshold,
|
||||
projects,
|
||||
};
|
||||
}
|
||||
|
||||
export function validateAutoSyncRemoteUrl(remoteUrl: string): void {
|
||||
const match = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim());
|
||||
if (!match) {
|
||||
throw new Error('must use git@github.com:owner/repo.git, git@gitlab.com:group/repo.git, or git@gitee.com:owner/repo.git');
|
||||
}
|
||||
const host = match[1].toLowerCase();
|
||||
const repoPath = match[2];
|
||||
if (!ALLOWED_REMOTE_HOSTS.has(host)) {
|
||||
throw new Error('host must be one of github.com, gitlab.com, or gitee.com');
|
||||
}
|
||||
if (repoPath.startsWith('/') || repoPath.includes('..') || repoPath.split('/').length < 2) {
|
||||
throw new Error('path must include owner/repo without traversal');
|
||||
}
|
||||
}
|
||||
|
||||
export function validateAutoSyncBranchName(branch: string): void {
|
||||
if (!branch.trim()) throw new Error('must not be empty');
|
||||
if (/[\s\0-\x1f\x7f]/.test(branch)) throw new Error('must not contain whitespace or control characters');
|
||||
if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref');
|
||||
if (branch.startsWith('-')) throw new Error('must not start with "-"');
|
||||
if (branch.includes('..')) throw new Error('must not contain ".."');
|
||||
if (branch.includes('`')) throw new Error('must not contain backticks');
|
||||
}
|
||||
|
||||
export function parseDurationMs(value: unknown): number {
|
||||
if (typeof value === 'number') return value * 1_000;
|
||||
const raw = String(value ?? '').trim();
|
||||
const match = /^(\d+)(ms|s|m)?$/.exec(raw);
|
||||
if (!match) return Number.NaN;
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2] ?? 's';
|
||||
if (unit === 'ms') return amount;
|
||||
if (unit === 's') return amount * 1_000;
|
||||
return amount * 60_000;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,29 @@
|
|||
export {
|
||||
AUTO_SYNC_CONFIG_FILE,
|
||||
AUTO_SYNC_FLAG,
|
||||
getAutoSyncConfigPath,
|
||||
loadAutoSyncConfig,
|
||||
parseAutoSyncConfig,
|
||||
parseAutoSyncFlag,
|
||||
parseBranchCandidates,
|
||||
parseDurationMs,
|
||||
validateAutoSyncBranchName,
|
||||
validateAutoSyncRemoteUrl,
|
||||
type AutoSyncConfig,
|
||||
type AutoSyncConfigLoadResult,
|
||||
type AutoSyncFlagDecision,
|
||||
type AutoSyncProjectConfig,
|
||||
} from './config.js';
|
||||
export {
|
||||
buildStateKey,
|
||||
getAutoSyncWatchDir,
|
||||
getAutoSyncStatePath,
|
||||
getProjectCommitInfoPath,
|
||||
loadAutoSyncState,
|
||||
saveAutoSyncState,
|
||||
shouldAnalyzeCommit,
|
||||
writeProjectCommitInfo,
|
||||
type AutoSyncAnalyzeStatus,
|
||||
type AutoSyncCommitState,
|
||||
type AutoSyncCommitStateEntry,
|
||||
type ProjectCommitInfoEntry,
|
||||
} from './state.js';
|
||||
export { extractRepoNameFromRemoteUrl } from './repo.js';
|
||||
export {
|
||||
|
|
@ -31,10 +35,19 @@ export {
|
|||
export {
|
||||
addRepoToGroup,
|
||||
getConfiguredRepoPath,
|
||||
resolveActualConcurrency,
|
||||
runAutoSyncOnce,
|
||||
syncGroupByName,
|
||||
type AutoSyncLogger,
|
||||
type AutoSyncRunDeps,
|
||||
type AutoSyncRunResult,
|
||||
} from './runner.js';
|
||||
export { maybeStartAutoSyncFromEnv, type AutoSyncStartHandle } from './starter.js';
|
||||
export {
|
||||
getAutoSyncWatchPaths,
|
||||
readAutoSyncWatchStatus,
|
||||
startAutoSyncWatch,
|
||||
stopAutoSyncWatch,
|
||||
type AutoSyncStartHandle,
|
||||
type AutoSyncWatchPaths,
|
||||
type WatchStatusRecord,
|
||||
} from './starter.js';
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
|
|||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { getGlobalDir } from '../../storage/repo-manager.js';
|
||||
import { getAutoSyncWatchDir } from './state.js';
|
||||
|
||||
const DANGEROUS_ROOTS = new Set(
|
||||
[
|
||||
|
|
@ -61,6 +62,7 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise<Aut
|
|||
const root = normalizeConfiguredCloneRoot(localPath);
|
||||
assertNotDangerousRoot(root);
|
||||
await assertNoSymlinkPath(root);
|
||||
await fs.mkdir(root, { recursive: true });
|
||||
await assertDirectoryOwnerAndPermissions(root);
|
||||
const realRoot = await fs.realpath(root);
|
||||
assertContainedOrSame(root, realRoot, 'Configured clone root realpath escaped its normalized path');
|
||||
|
|
@ -69,7 +71,7 @@ export async function resolveConfiguredCloneRoot(localPath: string): Promise<Aut
|
|||
|
||||
return {
|
||||
root: realRoot,
|
||||
quarantineRoot: path.join(getGlobalDir(), 'quarantine'),
|
||||
quarantineRoot: path.join(getAutoSyncWatchDir(), 'quarantine'),
|
||||
quarantineRetentionDays: QUARANTINE_RETENTION_DAYS,
|
||||
};
|
||||
}
|
||||
|
|
@ -123,10 +125,10 @@ function assertNotDangerousRoot(root: string): void {
|
|||
function assertNotGitNexusInternalRoot(root: string): void {
|
||||
const gitnexusDir = path.resolve(getGlobalDir());
|
||||
const blocked = [
|
||||
gitnexusDir,
|
||||
path.join(gitnexusDir, 'groups'),
|
||||
path.join(gitnexusDir, 'indexes'),
|
||||
path.join(gitnexusDir, 'quarantine'),
|
||||
path.join(getAutoSyncWatchDir(gitnexusDir), 'quarantine'),
|
||||
];
|
||||
for (const blockedRoot of blocked) {
|
||||
const rel = path.relative(blockedRoot, root);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { extractRepoName } from '../../server/git-clone.js';
|
||||
import { validateAutoSyncRemoteUrl } from './config.js';
|
||||
|
||||
export function extractRepoNameFromRemoteUrl(remoteUrl: string): string {
|
||||
validateAutoSyncRemoteUrl(remoteUrl);
|
||||
return extractRepoName(remoteUrl);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,13 @@ import {
|
|||
loadAutoSyncState,
|
||||
saveAutoSyncState,
|
||||
shouldAnalyzeCommit,
|
||||
writeProjectCommitInfo,
|
||||
type AutoSyncAnalyzeStatus,
|
||||
type AutoSyncCommitStateEntry,
|
||||
type ProjectCommitInfoEntry,
|
||||
} from './state.js';
|
||||
import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js';
|
||||
import { validateAutoSyncRemoteUrl } from './config.js';
|
||||
|
||||
export interface AutoSyncLogger {
|
||||
info(message: string): void;
|
||||
|
|
@ -33,9 +37,11 @@ export interface AutoSyncRunDeps {
|
|||
registerRepo: typeof registerRepo;
|
||||
loadState: typeof loadAutoSyncState;
|
||||
saveState: typeof saveAutoSyncState;
|
||||
writeCommitInfo: typeof writeProjectCommitInfo;
|
||||
addRepoToGroup: typeof addRepoToGroup;
|
||||
syncGroupByName: typeof syncGroupByName;
|
||||
resolveCloneRoot: typeof resolveConfiguredCloneRoot;
|
||||
getAvailableMemoryGB: () => number;
|
||||
}
|
||||
|
||||
export interface AutoSyncRunResult {
|
||||
|
|
@ -59,9 +65,11 @@ const DEFAULT_DEPS: AutoSyncRunDeps = {
|
|||
registerRepo,
|
||||
loadState: loadAutoSyncState,
|
||||
saveState: saveAutoSyncState,
|
||||
writeCommitInfo: writeProjectCommitInfo,
|
||||
addRepoToGroup,
|
||||
syncGroupByName,
|
||||
resolveCloneRoot: resolveConfiguredCloneRoot,
|
||||
getAvailableMemoryGB: () => Math.floor(process.availableMemory?.() ?? 0) / 1024 / 1024 / 1024,
|
||||
};
|
||||
|
||||
export async function runAutoSyncOnce(
|
||||
|
|
@ -74,89 +82,192 @@ export async function runAutoSyncOnce(
|
|||
const state = await deps.loadState();
|
||||
const groupsToSync = new Set<string>();
|
||||
const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 };
|
||||
const commitInfoEntries: ProjectCommitInfoEntry[] = [];
|
||||
const actualConcurrency = resolveActualConcurrency(config.maxConcurrency, deps.getAvailableMemoryGB());
|
||||
logger.info(
|
||||
`[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`,
|
||||
);
|
||||
|
||||
for (const project of config.projects) {
|
||||
for (const remoteUrl of project.remoteUrls) {
|
||||
try {
|
||||
const repoName = extractRepoNameFromRemoteUrl(remoteUrl);
|
||||
const cloneRoot = await deps.resolveCloneRoot(project.localPath);
|
||||
const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName);
|
||||
await deps.cloneOrPull(remoteUrl, targetDir, undefined, {
|
||||
allowedCloneRoot: cloneRoot.root,
|
||||
expectedRepoName: repoName,
|
||||
quarantineRoot: cloneRoot.quarantineRoot,
|
||||
});
|
||||
result.synced += 1;
|
||||
|
||||
const currentBranch = deps.getCurrentBranch(targetDir);
|
||||
if (!currentBranch || !project.branches.includes(currentBranch)) {
|
||||
result.skippedAnalysis += 1;
|
||||
logger.warn(
|
||||
`[auto-sync] Skip analysis for ${targetDir}; current branch ${currentBranch ?? '<detached>'} is not in configured branches: ${project.branches.join(', ')}.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const branch = currentBranch;
|
||||
const currentCommit = deps.getCurrentCommit(targetDir);
|
||||
const stateKey = buildStateKey(targetDir, branch);
|
||||
const previous = state[stateKey];
|
||||
let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped';
|
||||
let analyzedCommitId = previous?.analyzedCommitId;
|
||||
|
||||
if (
|
||||
shouldAnalyzeCommit({
|
||||
currentCommit,
|
||||
previousAnalyzedCommit: previous?.analyzedCommitId,
|
||||
previousStatus: previous?.lastAnalyzeStatus,
|
||||
})
|
||||
) {
|
||||
try {
|
||||
const analysis = await deps.runFullAnalysis(
|
||||
targetDir,
|
||||
{ branch, skipAgentsMd: true, skipSkills: true },
|
||||
{ onProgress: () => {} },
|
||||
);
|
||||
const meta: RepoMeta = {
|
||||
repoPath: targetDir,
|
||||
lastCommit: currentCommit,
|
||||
indexedAt: now().toISOString(),
|
||||
stats: analysis.stats,
|
||||
branch,
|
||||
};
|
||||
await deps.registerRepo(targetDir, meta, { name: repoName, allowDuplicateName: true });
|
||||
analyzeStatus = 'success';
|
||||
analyzedCommitId = currentCommit;
|
||||
result.analyzed += 1;
|
||||
} catch (err: unknown) {
|
||||
analyzeStatus = 'failed';
|
||||
result.failed += 1;
|
||||
logger.error(`[auto-sync] Analysis failed for ${targetDir}: ${(err as Error).message}`);
|
||||
}
|
||||
} else {
|
||||
result.skippedAnalysis += 1;
|
||||
logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`);
|
||||
}
|
||||
|
||||
state[stateKey] = {
|
||||
codeCommitId: currentCommit,
|
||||
analyzedCommitId,
|
||||
lastAnalyzeStatus: analyzeStatus,
|
||||
lastSyncTime: now().toISOString(),
|
||||
const workItems = await buildWorkItems(config, deps);
|
||||
const repoResults = await mapWithConcurrency(workItems, actualConcurrency, async (item) => {
|
||||
const lastSyncTime = now().toISOString();
|
||||
try {
|
||||
validateAutoSyncRemoteUrl(item.remoteUrl);
|
||||
const repoName = extractRepoNameFromRemoteUrl(item.remoteUrl);
|
||||
const targetDir = getConfiguredRepoPath({ localPath: item.cloneRoot.root }, repoName);
|
||||
const syncResult = await syncFirstAvailableBranch({
|
||||
item,
|
||||
repoName,
|
||||
targetDir,
|
||||
timeoutMs: config.repoGitTimeoutMs,
|
||||
deps,
|
||||
logger,
|
||||
});
|
||||
if (syncResult.ok === false) {
|
||||
logger.error(
|
||||
`[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`,
|
||||
);
|
||||
return {
|
||||
kind: 'failed' as const,
|
||||
project: item.project,
|
||||
remoteUrl: item.remoteUrl,
|
||||
targetDir,
|
||||
branch: item.project.branches[0],
|
||||
status: syncResult.status,
|
||||
analyzeConsecutiveFailures: 0,
|
||||
lastSyncTime,
|
||||
};
|
||||
}
|
||||
|
||||
if (project.gitnexusGroup) {
|
||||
const added = await deps.addRepoToGroup(project, repoName);
|
||||
if (added) groupsToSync.add(project.gitnexusGroup);
|
||||
const currentBranch = syncResult.branch;
|
||||
|
||||
const currentCommit = deps.getCurrentCommit(targetDir);
|
||||
const stateKey = buildStateKey(targetDir, currentBranch);
|
||||
const previous = state[stateKey];
|
||||
let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped';
|
||||
let analyzedCommitId = previous?.analyzedCommitId;
|
||||
let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0;
|
||||
let lastAnalyzeError = previous?.lastAnalyzeError;
|
||||
let stats: RepoMeta['stats'] | undefined;
|
||||
|
||||
if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) {
|
||||
analyzeStatus = 'threshold_skipped';
|
||||
logger.error(
|
||||
`[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`,
|
||||
);
|
||||
} else if (
|
||||
shouldAnalyzeCommit({
|
||||
currentCommit,
|
||||
previousAnalyzedCommit: previous?.analyzedCommitId,
|
||||
previousStatus: previous?.lastAnalyzeStatus,
|
||||
})
|
||||
) {
|
||||
try {
|
||||
const analysis = await deps.runFullAnalysis(
|
||||
targetDir,
|
||||
{ branch: currentBranch, skipAgentsMd: true, skipSkills: true },
|
||||
{ onProgress: () => {} },
|
||||
);
|
||||
stats = analysis.stats;
|
||||
analyzeStatus = 'success';
|
||||
analyzedCommitId = currentCommit;
|
||||
analyzeConsecutiveFailures = 0;
|
||||
lastAnalyzeError = undefined;
|
||||
} catch (err: unknown) {
|
||||
analyzeStatus = 'failed';
|
||||
analyzeConsecutiveFailures += 1;
|
||||
lastAnalyzeError = shortErrorMessage(err);
|
||||
logger.error(
|
||||
`[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`);
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'synced' as const,
|
||||
project: item.project,
|
||||
repoName,
|
||||
remoteUrl: item.remoteUrl,
|
||||
targetDir,
|
||||
branch: currentBranch,
|
||||
currentCommit,
|
||||
analyzedCommitId,
|
||||
analyzeStatus,
|
||||
analyzeConsecutiveFailures,
|
||||
lastAnalyzeError,
|
||||
stats,
|
||||
stateKey,
|
||||
lastSyncTime,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
logger.error(`[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`);
|
||||
return {
|
||||
kind: 'failed' as const,
|
||||
project: item.project,
|
||||
remoteUrl: item.remoteUrl,
|
||||
targetDir: '',
|
||||
status: 'sync_failed' as const,
|
||||
lastSyncTime,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
for (const repoResult of repoResults) {
|
||||
if (repoResult.kind === 'failed') {
|
||||
result.failed += 1;
|
||||
commitInfoEntries.push({
|
||||
remoteUrl: repoResult.remoteUrl,
|
||||
localPath: repoResult.targetDir,
|
||||
branch: repoResult.branch,
|
||||
status: repoResult.status,
|
||||
lastSyncTime: repoResult.lastSyncTime,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
result.synced += 1;
|
||||
const stateEntry: AutoSyncCommitStateEntry = {
|
||||
codeCommitId: repoResult.currentCommit,
|
||||
analyzedCommitId: repoResult.analyzedCommitId,
|
||||
lastAnalyzeStatus: repoResult.analyzeStatus,
|
||||
analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures,
|
||||
lastAnalyzeError: repoResult.lastAnalyzeError,
|
||||
lastSyncTime: repoResult.lastSyncTime,
|
||||
};
|
||||
state[repoResult.stateKey] = stateEntry;
|
||||
if (repoResult.analyzeStatus === 'success') {
|
||||
const meta: RepoMeta = {
|
||||
repoPath: repoResult.targetDir,
|
||||
lastCommit: repoResult.currentCommit,
|
||||
indexedAt: repoResult.lastSyncTime,
|
||||
stats: repoResult.stats!,
|
||||
branch: repoResult.branch,
|
||||
};
|
||||
await deps.registerRepo(repoResult.targetDir, meta, {
|
||||
name: repoResult.repoName,
|
||||
allowDuplicateName: true,
|
||||
});
|
||||
result.analyzed += 1;
|
||||
} else if (repoResult.analyzeStatus === 'failed') {
|
||||
result.failed += 1;
|
||||
} else if (repoResult.analyzeStatus === 'threshold_skipped') {
|
||||
result.skippedAnalysis += 1;
|
||||
} else {
|
||||
result.skippedAnalysis += 1;
|
||||
}
|
||||
|
||||
commitInfoEntries.push({
|
||||
remoteUrl: repoResult.remoteUrl,
|
||||
localPath: repoResult.targetDir,
|
||||
branch: repoResult.branch,
|
||||
codeCommitId: repoResult.currentCommit,
|
||||
analyzedCommitId: repoResult.analyzedCommitId,
|
||||
status: repoResult.analyzeStatus,
|
||||
analyzeConsecutiveFailures: repoResult.analyzeConsecutiveFailures,
|
||||
analyzeFailureThreshold: config.analyzeFailureThreshold,
|
||||
lastAnalyzeError: repoResult.lastAnalyzeError,
|
||||
lastSyncTime: repoResult.lastSyncTime,
|
||||
});
|
||||
|
||||
if (repoResult.project.groupName) {
|
||||
let groupMembershipOk = false;
|
||||
try {
|
||||
await deps.addRepoToGroup(repoResult.project, repoResult.repoName);
|
||||
groupMembershipOk = true;
|
||||
} catch (err: unknown) {
|
||||
result.failed += 1;
|
||||
logger.error(`[auto-sync] Repository sync failed for ${remoteUrl}: ${(err as Error).message}`);
|
||||
logger.error(`[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`);
|
||||
}
|
||||
if (groupMembershipOk && repoResult.analyzeStatus === 'success') {
|
||||
groupsToSync.add(repoResult.project.groupName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await deps.saveState(state);
|
||||
await deps.writeCommitInfo(commitInfoEntries);
|
||||
for (const groupName of groupsToSync) {
|
||||
try {
|
||||
await deps.syncGroupByName(groupName);
|
||||
|
|
@ -168,6 +279,11 @@ export async function runAutoSyncOnce(
|
|||
return result;
|
||||
}
|
||||
|
||||
function shortErrorMessage(err: unknown): string {
|
||||
const message = (err as Error).message || String(err);
|
||||
return message.replace(/\s+/g, ' ').slice(0, 240);
|
||||
}
|
||||
|
||||
export function getConfiguredRepoPath(
|
||||
project: Pick<AutoSyncProjectConfig, 'localPath'>,
|
||||
repoName: string,
|
||||
|
|
@ -176,11 +292,11 @@ export function getConfiguredRepoPath(
|
|||
}
|
||||
|
||||
export async function addRepoToGroup(
|
||||
project: Pick<AutoSyncProjectConfig, 'gitnexusGroup'>,
|
||||
project: Pick<AutoSyncProjectConfig, 'groupName'>,
|
||||
repoName: string,
|
||||
): Promise<boolean> {
|
||||
if (!project.gitnexusGroup) return false;
|
||||
const groupDir = getGroupDir(getDefaultGitnexusDir(), project.gitnexusGroup);
|
||||
if (!project.groupName) return false;
|
||||
const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName);
|
||||
const config = await loadGroupConfig(groupDir);
|
||||
if (Object.values(config.repos).includes(repoName)) return false;
|
||||
config.repos[repoName] = repoName;
|
||||
|
|
@ -199,3 +315,100 @@ async function writeGroupConfigAtomic(filePath: string, config: unknown): Promis
|
|||
await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8');
|
||||
await fs.rename(tmpPath, filePath);
|
||||
}
|
||||
|
||||
export function resolveActualConcurrency(configured: number, availableMemoryGB: number): number {
|
||||
const memoryLimit = Math.max(1, Math.floor(availableMemoryGB / 2));
|
||||
return Math.max(1, Math.min(configured, memoryLimit));
|
||||
}
|
||||
|
||||
async function buildWorkItems(config: AutoSyncConfig, deps: AutoSyncRunDeps): Promise<AutoSyncWorkItem[]> {
|
||||
const items: AutoSyncWorkItem[] = [];
|
||||
const targetOwners = new Map<string, string>();
|
||||
for (const project of config.projects) {
|
||||
const cloneRoot = await deps.resolveCloneRoot(project.localPath);
|
||||
for (const remoteUrl of project.remoteUrls) {
|
||||
try {
|
||||
const repoName = extractRepoNameFromRemoteUrl(remoteUrl);
|
||||
const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName);
|
||||
const previous = targetOwners.get(targetDir);
|
||||
if (previous !== undefined) {
|
||||
throw new Error(`Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`);
|
||||
}
|
||||
targetOwners.set(targetDir, remoteUrl);
|
||||
} catch (err: unknown) {
|
||||
if ((err as Error).message.startsWith('Duplicate auto-sync targetDir')) throw err;
|
||||
}
|
||||
items.push({ project, remoteUrl, cloneRoot });
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
||||
while (nextIndex < items.length) {
|
||||
const currentIndex = nextIndex;
|
||||
nextIndex += 1;
|
||||
results[currentIndex] = await worker(items[currentIndex]);
|
||||
}
|
||||
});
|
||||
await Promise.all(runners);
|
||||
return results;
|
||||
}
|
||||
|
||||
interface AutoSyncWorkItem {
|
||||
project: AutoSyncProjectConfig;
|
||||
remoteUrl: string;
|
||||
cloneRoot: Awaited<ReturnType<typeof resolveConfiguredCloneRoot>>;
|
||||
}
|
||||
|
||||
async function syncFirstAvailableBranch(input: {
|
||||
item: AutoSyncWorkItem;
|
||||
repoName: string;
|
||||
targetDir: string;
|
||||
timeoutMs: number;
|
||||
deps: AutoSyncRunDeps;
|
||||
logger: AutoSyncLogger;
|
||||
}): Promise<
|
||||
| { ok: true; branch: string }
|
||||
| { ok: false; status: 'branch_unavailable' | 'sync_timeout'; message: string }
|
||||
> {
|
||||
const failures: string[] = [];
|
||||
let sawTimeout = false;
|
||||
for (const branch of input.item.project.branches) {
|
||||
try {
|
||||
await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, {
|
||||
allowedCloneRoot: input.item.cloneRoot.root,
|
||||
expectedRepoName: input.repoName,
|
||||
quarantineRoot: input.item.cloneRoot.quarantineRoot,
|
||||
allowAutoSyncSsh: true,
|
||||
timeoutMs: input.timeoutMs,
|
||||
branch,
|
||||
});
|
||||
const currentBranch = input.deps.getCurrentBranch(input.targetDir);
|
||||
if (currentBranch === branch) return { ok: true, branch };
|
||||
failures.push(`${branch}: checked out ${currentBranch ?? '<detached>'}`);
|
||||
input.logger.warn(
|
||||
`[auto-sync] Branch ${branch} for ${input.item.remoteUrl} synced but current branch is ${currentBranch ?? '<detached>'}; trying next branch.`,
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
const message = (err as Error).message;
|
||||
if (message.includes('timed out')) sawTimeout = true;
|
||||
failures.push(`${branch}: ${message}`);
|
||||
input.logger.warn(
|
||||
`[auto-sync] Branch ${branch} unavailable for ${input.item.remoteUrl}: ${message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
status: sawTimeout ? 'sync_timeout' : 'branch_unavailable',
|
||||
message: failures.join('; '),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,28 +1,95 @@
|
|||
import { loadAutoSyncConfig, parseAutoSyncFlag } from './config.js';
|
||||
import fs from 'node:fs/promises';
|
||||
import crypto from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { getGlobalDir } from '../../storage/repo-manager.js';
|
||||
import { loadAutoSyncConfig } from './config.js';
|
||||
import { runAutoSyncOnce } from './runner.js';
|
||||
import { getAutoSyncWatchDir } from './state.js';
|
||||
|
||||
export interface AutoSyncStartHandle {
|
||||
stop(): void;
|
||||
stop(): Promise<void>;
|
||||
}
|
||||
|
||||
export async function maybeStartAutoSyncFromEnv(options: {
|
||||
export type WatchStatusState = 'running' | 'stopping' | 'stopped' | 'stale' | 'error';
|
||||
|
||||
export interface WatchStatusRecord {
|
||||
state: WatchStatusState;
|
||||
pid?: number;
|
||||
ownerId?: string;
|
||||
configPath?: string;
|
||||
message?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface WatchLockRecord {
|
||||
pid: number;
|
||||
ownerId: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface AutoSyncWatchPaths {
|
||||
pidPath: string;
|
||||
lockPath: string;
|
||||
statusPath: string;
|
||||
}
|
||||
|
||||
export interface AutoSyncWatchControlDeps {
|
||||
isProcessAlive(pid: number): boolean;
|
||||
killProcess(pid: number, signal?: NodeJS.Signals): void;
|
||||
sleep(ms: number): Promise<void>;
|
||||
}
|
||||
|
||||
export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWatchPaths {
|
||||
const watchDir = getAutoSyncWatchDir(gitnexusDir);
|
||||
return {
|
||||
pidPath: path.join(watchDir, 'watch.pid'),
|
||||
lockPath: path.join(watchDir, 'watch.lock'),
|
||||
statusPath: path.join(watchDir, 'watch.status.json'),
|
||||
};
|
||||
}
|
||||
|
||||
export async function startAutoSyncWatch(options: {
|
||||
setIntervalFn?: typeof setInterval;
|
||||
clearIntervalFn?: typeof clearInterval;
|
||||
runOnce?: typeof runAutoSyncOnce;
|
||||
stderr?: Pick<NodeJS.WriteStream, 'write'>;
|
||||
keepAlive?: boolean;
|
||||
paths?: AutoSyncWatchPaths;
|
||||
deps?: Partial<AutoSyncWatchControlDeps>;
|
||||
} = {}): Promise<AutoSyncStartHandle | null> {
|
||||
const stderr = options.stderr ?? process.stderr;
|
||||
const flag = parseAutoSyncFlag();
|
||||
if (flag.enabled === false) {
|
||||
if (flag.message) stderr.write(`${flag.message}\n`);
|
||||
return null;
|
||||
}
|
||||
const paths = options.paths ?? getAutoSyncWatchPaths();
|
||||
const deps = resolveWatchDeps(options.deps);
|
||||
const ownerId = crypto.randomUUID();
|
||||
await fs.mkdir(path.dirname(paths.pidPath), { recursive: true });
|
||||
const lockHandle = await acquireWatchLock(paths, deps, stderr);
|
||||
if (!lockHandle) return null;
|
||||
await lockHandle.writeFile(
|
||||
`${JSON.stringify({ pid: process.pid, ownerId, createdAt: new Date().toISOString() })}\n`,
|
||||
'utf-8',
|
||||
);
|
||||
await fs.writeFile(paths.pidPath, `${process.pid}\n`, 'utf-8');
|
||||
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
if (loaded.ok === false) {
|
||||
stderr.write(`${loaded.message}\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
pid: process.pid,
|
||||
ownerId,
|
||||
message: loaded.message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
await cleanupWatchFiles(paths, lockHandle);
|
||||
return null;
|
||||
}
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'running',
|
||||
pid: process.pid,
|
||||
ownerId,
|
||||
configPath: loaded.config.configPath,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const runOnce = options.runOnce ?? runAutoSyncOnce;
|
||||
let running = false;
|
||||
|
|
@ -32,9 +99,17 @@ export async function maybeStartAutoSyncFromEnv(options: {
|
|||
return;
|
||||
}
|
||||
running = true;
|
||||
const startedAt = new Date();
|
||||
stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`);
|
||||
void runOnce(loaded.config)
|
||||
.then((result) => {
|
||||
stderr.write(
|
||||
`[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`,
|
||||
);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`);
|
||||
stderr.write('[auto-sync] Watch loop finished: failed.\n');
|
||||
})
|
||||
.finally(() => {
|
||||
running = false;
|
||||
|
|
@ -46,6 +121,359 @@ export async function maybeStartAutoSyncFromEnv(options: {
|
|||
const setIntervalFn = options.setIntervalFn ?? setInterval;
|
||||
const clearIntervalFn = options.clearIntervalFn ?? clearInterval;
|
||||
const timer = setIntervalFn(runSafely, intervalMs);
|
||||
timer.unref?.();
|
||||
return { stop: () => clearIntervalFn(timer) };
|
||||
if (options.keepAlive === false) timer.unref?.();
|
||||
return {
|
||||
stop: async () => {
|
||||
clearIntervalFn(timer);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stopped',
|
||||
pid: process.pid,
|
||||
ownerId,
|
||||
configPath: loaded.config.configPath,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).finally(() => cleanupWatchFiles(paths, lockHandle));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function acquireWatchLock(
|
||||
paths: AutoSyncWatchPaths,
|
||||
deps: AutoSyncWatchControlDeps,
|
||||
stderr: Pick<NodeJS.WriteStream, 'write'>,
|
||||
): Promise<fs.FileHandle | null> {
|
||||
try {
|
||||
return await fs.open(paths.lockPath, 'wx');
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
|
||||
}
|
||||
|
||||
const lock = await readLockFile(paths.lockPath);
|
||||
if (!lock) {
|
||||
const message = 'watch lock already exists but has no readable owner; refusing to start';
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
if (deps.isProcessAlive(lock.pid)) {
|
||||
stderr.write(`[auto-sync] Watch is already running with pid ${lock.pid}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'running',
|
||||
pid: lock.pid,
|
||||
ownerId: lock.ownerId,
|
||||
message: 'watch already running',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`);
|
||||
await removeIfExists(paths.pidPath);
|
||||
await removeIfExists(paths.lockPath);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stale',
|
||||
pid: lock.pid,
|
||||
ownerId: lock.ownerId,
|
||||
message: 'removed stale lock and pid',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
return await fs.open(paths.lockPath, 'wx');
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'EEXIST') {
|
||||
const message = 'watch lock was reacquired by another process; refusing to start';
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function stopAutoSyncWatch(options: {
|
||||
paths?: AutoSyncWatchPaths;
|
||||
stderr?: Pick<NodeJS.WriteStream, 'write'>;
|
||||
deps?: Partial<AutoSyncWatchControlDeps>;
|
||||
timeoutMs?: number;
|
||||
pollMs?: number;
|
||||
} = {}): Promise<boolean> {
|
||||
const stderr = options.stderr ?? process.stderr;
|
||||
const paths = options.paths ?? getAutoSyncWatchPaths();
|
||||
const deps = resolveWatchDeps(options.deps);
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const pollMs = options.pollMs ?? 100;
|
||||
const pid = await readPid(paths.pidPath);
|
||||
if (!pid) {
|
||||
const lock = await readLockFile(paths.lockPath);
|
||||
if (lock && deps.isProcessAlive(lock.pid)) {
|
||||
const message = `watch appears to be starting with pid ${lock.pid}; pid file is not ready`;
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
pid: lock.pid,
|
||||
ownerId: lock.ownerId,
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (lock) {
|
||||
stderr.write(`[auto-sync] Removing stale watch lock for pid ${lock.pid}.\n`);
|
||||
await removeIfExists(paths.lockPath);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stale',
|
||||
pid: lock.pid,
|
||||
ownerId: lock.ownerId,
|
||||
message: 'removed stale lock without pid file',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (await fileExists(paths.lockPath)) {
|
||||
const message = 'watch lock exists but has no readable owner; refusing to stop';
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
stderr.write('[auto-sync] Watch is not running.\n');
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stopped',
|
||||
message: 'no pid file',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (!deps.isProcessAlive(pid)) {
|
||||
stderr.write(`[auto-sync] Removing stale watch pid ${pid}.\n`);
|
||||
await removeIfExists(paths.pidPath);
|
||||
await removeIfExists(paths.lockPath);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stale',
|
||||
pid,
|
||||
message: 'removed stale pid and lock',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
const owner = await readVerifiedWatchOwner(paths, pid);
|
||||
if (owner.ok === false) {
|
||||
const message = `refusing to stop pid ${pid}; ${owner.reason}`;
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'error',
|
||||
pid,
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stopping',
|
||||
pid,
|
||||
ownerId: owner.owner.ownerId,
|
||||
message: 'stop signal sent; waiting for watch process to exit',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
deps.killProcess(pid, 'SIGTERM');
|
||||
stderr.write(`[auto-sync] Stop signal sent to watch pid ${pid}.\n`);
|
||||
const stopped = await waitForProcessExit(pid, { deps, timeoutMs, pollMs });
|
||||
if (!stopped) {
|
||||
const message = `watch pid ${pid} did not exit within ${timeoutMs}ms`;
|
||||
stderr.write(`[auto-sync] ${message}.\n`);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stopping',
|
||||
pid,
|
||||
ownerId: owner.owner.ownerId,
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
await removeIfExists(paths.pidPath);
|
||||
await removeIfExists(paths.lockPath);
|
||||
await writeWatchStatus(paths, {
|
||||
state: 'stopped',
|
||||
pid,
|
||||
ownerId: owner.owner.ownerId,
|
||||
message: 'watch stopped',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function readAutoSyncWatchStatus(
|
||||
paths = getAutoSyncWatchPaths(),
|
||||
deps: Partial<AutoSyncWatchControlDeps> = {},
|
||||
): Promise<WatchStatusRecord> {
|
||||
const resolvedDeps = resolveWatchDeps(deps);
|
||||
const pid = await readPid(paths.pidPath);
|
||||
if (pid && !resolvedDeps.isProcessAlive(pid)) {
|
||||
return {
|
||||
state: 'stale',
|
||||
pid,
|
||||
message: 'pid file exists but process is not running',
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
if (pid) {
|
||||
const stored = await readStatusFile(paths.statusPath);
|
||||
const owner = await readVerifiedWatchOwner(paths, pid);
|
||||
if (owner.ok === false) {
|
||||
return {
|
||||
...stored,
|
||||
state: 'error',
|
||||
pid,
|
||||
message: owner.reason,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
return {
|
||||
...stored,
|
||||
state: stored?.state === 'stopping' ? 'stopping' : 'running',
|
||||
pid,
|
||||
ownerId: owner.owner.ownerId,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
const stored = await readStatusFile(paths.statusPath);
|
||||
return stored ?? { state: 'stopped', updatedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
async function readLockFile(lockPath: string): Promise<WatchLockRecord | undefined> {
|
||||
try {
|
||||
const raw = await fs.readFile(lockPath, 'utf-8');
|
||||
const parsed = JSON.parse(raw) as WatchLockRecord;
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === 'object' &&
|
||||
Number.isInteger(parsed.pid) &&
|
||||
parsed.pid > 0 &&
|
||||
typeof parsed.ownerId === 'string' &&
|
||||
parsed.ownerId
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
return undefined;
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function readVerifiedWatchOwner(
|
||||
paths: AutoSyncWatchPaths,
|
||||
pid: number,
|
||||
): Promise<{ ok: true; owner: WatchLockRecord } | { ok: false; reason: string }> {
|
||||
const [status, lock] = await Promise.all([
|
||||
readStatusFile(paths.statusPath),
|
||||
readLockFile(paths.lockPath),
|
||||
]);
|
||||
if (!lock) return { ok: false, reason: 'watch lock is missing or invalid' };
|
||||
if (!status) return { ok: false, reason: 'watch status is missing or invalid' };
|
||||
if (lock.pid !== pid) return { ok: false, reason: 'watch lock pid does not match pid file' };
|
||||
if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' };
|
||||
if (!status.ownerId || status.ownerId !== lock.ownerId) {
|
||||
return { ok: false, reason: 'watch status owner does not match lock owner' };
|
||||
}
|
||||
return { ok: true, owner: lock };
|
||||
}
|
||||
|
||||
async function waitForProcessExit(
|
||||
pid: number,
|
||||
options: { deps: AutoSyncWatchControlDeps; timeoutMs: number; pollMs: number },
|
||||
): Promise<boolean> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (!options.deps.isProcessAlive(pid)) return true;
|
||||
await options.deps.sleep(options.pollMs);
|
||||
}
|
||||
return !options.deps.isProcessAlive(pid);
|
||||
}
|
||||
|
||||
async function readPid(pidPath: string): Promise<number | undefined> {
|
||||
try {
|
||||
const raw = await fs.readFile(pidPath, 'utf-8');
|
||||
const pid = Number(raw.trim());
|
||||
return Number.isInteger(pid) && pid > 0 ? pid : undefined;
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function readStatusFile(statusPath: string): Promise<WatchStatusRecord | undefined> {
|
||||
try {
|
||||
const parsed = JSON.parse(await fs.readFile(statusPath, 'utf-8')) as WatchStatusRecord;
|
||||
return parsed && typeof parsed === 'object' ? parsed : undefined;
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
return {
|
||||
state: 'error',
|
||||
message: `unable to read status file: ${(err as Error).message}`,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function writeWatchStatus(paths: AutoSyncWatchPaths, record: WatchStatusRecord): Promise<void> {
|
||||
await fs.mkdir(path.dirname(paths.statusPath), { recursive: true });
|
||||
const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8');
|
||||
await fs.rename(tmpPath, paths.statusPath);
|
||||
}
|
||||
|
||||
async function cleanupWatchFiles(paths: AutoSyncWatchPaths, lockHandle?: fs.FileHandle): Promise<void> {
|
||||
await lockHandle?.close().catch(() => {});
|
||||
await removeIfExists(paths.pidPath);
|
||||
await removeIfExists(paths.lockPath);
|
||||
}
|
||||
|
||||
async function removeIfExists(filePath: string): Promise<void> {
|
||||
await fs.rm(filePath, { force: true });
|
||||
}
|
||||
|
||||
async function fileExists(filePath: string): Promise<boolean> {
|
||||
return fs.access(filePath).then(
|
||||
() => true,
|
||||
() => false,
|
||||
);
|
||||
}
|
||||
|
||||
function resolveWatchDeps(deps: Partial<AutoSyncWatchControlDeps> = {}): AutoSyncWatchControlDeps {
|
||||
return {
|
||||
isProcessAlive:
|
||||
deps.isProcessAlive ??
|
||||
((pid) => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
killProcess:
|
||||
deps.killProcess ??
|
||||
((pid, signal = 'SIGTERM') => {
|
||||
process.kill(pid, signal);
|
||||
}),
|
||||
sleep:
|
||||
deps.sleep ??
|
||||
((ms) =>
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,19 +2,29 @@ import fs from 'node:fs/promises';
|
|||
import path from 'node:path';
|
||||
import { getGlobalDir } from '../../storage/repo-manager.js';
|
||||
|
||||
export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped';
|
||||
export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped';
|
||||
|
||||
export interface AutoSyncCommitStateEntry {
|
||||
codeCommitId: string;
|
||||
analyzedCommitId?: string;
|
||||
lastAnalyzeStatus?: AutoSyncAnalyzeStatus;
|
||||
analyzeConsecutiveFailures?: number;
|
||||
lastAnalyzeError?: string;
|
||||
lastSyncTime: string;
|
||||
}
|
||||
|
||||
export type AutoSyncCommitState = Record<string, AutoSyncCommitStateEntry>;
|
||||
|
||||
export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string {
|
||||
return path.join(gitnexusDir, 'watch');
|
||||
}
|
||||
|
||||
export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string {
|
||||
return path.join(gitnexusDir, 'auto-sync-state.json');
|
||||
return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json');
|
||||
}
|
||||
|
||||
export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string {
|
||||
return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt');
|
||||
}
|
||||
|
||||
export function buildStateKey(repoPath: string, branch: string): string {
|
||||
|
|
@ -59,3 +69,46 @@ export async function saveAutoSyncState(
|
|||
await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8');
|
||||
await fs.rename(tmpPath, statePath);
|
||||
}
|
||||
|
||||
export async function writeProjectCommitInfo(
|
||||
entries: ProjectCommitInfoEntry[],
|
||||
infoPath = getProjectCommitInfoPath(),
|
||||
): Promise<void> {
|
||||
await fs.mkdir(path.dirname(infoPath), { recursive: true });
|
||||
const lines = [
|
||||
'# GitNexus auto-sync project commit info',
|
||||
`updated_at: ${new Date().toISOString()}`,
|
||||
'',
|
||||
...entries.flatMap((entry) => [
|
||||
`remote: ${entry.remoteUrl}`,
|
||||
`local_path: ${entry.localPath}`,
|
||||
`branch: ${entry.branch ?? ''}`,
|
||||
`code_commit: ${entry.codeCommitId ?? ''}`,
|
||||
`analyzed_commit: ${entry.analyzedCommitId ?? ''}`,
|
||||
`status: ${entry.status}`,
|
||||
`analyze_consecutive_failures: ${entry.analyzeConsecutiveFailures ?? 0}`,
|
||||
...(entry.analyzeFailureThreshold === undefined
|
||||
? []
|
||||
: [`analyze_failure_threshold: ${entry.analyzeFailureThreshold}`]),
|
||||
...(entry.lastAnalyzeError ? [`last_analyze_error: ${entry.lastAnalyzeError}`] : []),
|
||||
`last_sync_time: ${entry.lastSyncTime}`,
|
||||
'',
|
||||
]),
|
||||
];
|
||||
const tmpPath = `${infoPath}.tmp.${process.pid}.${Date.now()}`;
|
||||
await fs.writeFile(tmpPath, `${lines.join('\n')}\n`, 'utf-8');
|
||||
await fs.rename(tmpPath, infoPath);
|
||||
}
|
||||
|
||||
export interface ProjectCommitInfoEntry {
|
||||
remoteUrl: string;
|
||||
localPath: string;
|
||||
branch?: string;
|
||||
codeCommitId?: string;
|
||||
analyzedCommitId?: string;
|
||||
status: AutoSyncAnalyzeStatus | 'sync_failed' | 'branch_skipped' | 'branch_unavailable' | 'sync_timeout';
|
||||
analyzeConsecutiveFailures?: number;
|
||||
analyzeFailureThreshold?: number;
|
||||
lastAnalyzeError?: string;
|
||||
lastSyncTime: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ import { fileURLToPath } from 'url';
|
|||
import { JobManager } from './analyze-job.js';
|
||||
import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js';
|
||||
import {
|
||||
extractRepoName,
|
||||
extractWebRepoName,
|
||||
getCloneDir,
|
||||
cloneOrPull,
|
||||
warnIfInsecureAzureConfig,
|
||||
|
|
@ -1582,7 +1582,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
try {
|
||||
// Clone if URL provided
|
||||
if (repoUrl && !repoLocalPath) {
|
||||
const repoName = extractRepoName(repoUrl);
|
||||
const repoName = extractWebRepoName(repoUrl);
|
||||
targetPath = getCloneDir(repoName);
|
||||
|
||||
jobManager.updateJob(job.id, {
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ import fs from 'fs/promises';
|
|||
import { isIP } from 'net';
|
||||
import { logger } from '../core/logger.js';
|
||||
import { getGlobalDir } from '../storage/repo-manager.js';
|
||||
import { sanitizeRepoName } from '../storage/git.js';
|
||||
import {
|
||||
assertDirectoryOwnerAndPermissions,
|
||||
quarantineAutoSyncPartial,
|
||||
} from '../core/auto-sync/path-security.js';
|
||||
import { validateAutoSyncRemoteUrl } from '../core/auto-sync/config.js';
|
||||
|
||||
/**
|
||||
* Root directory for all cloned repositories. Targets must resolve inside this.
|
||||
|
|
@ -59,6 +61,26 @@ export function extractRepoName(url: string): string {
|
|||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a clone directory name for the web `/api/analyze` boundary.
|
||||
*
|
||||
* The API historically accepted Azure DevOps and similar URLs whose repo
|
||||
* segment contains spaces or other directory-unsafe characters by sanitizing
|
||||
* the final segment. Keep that compatibility at the web boundary while leaving
|
||||
* `extractRepoName()` strict for internal/security-sensitive callers.
|
||||
*/
|
||||
export function extractWebRepoName(url: string): string {
|
||||
let trimmed = url.trim();
|
||||
while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1);
|
||||
const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed;
|
||||
const rawName = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? '';
|
||||
const safeName = sanitizeRepoName(rawName);
|
||||
if (!rawName || safeName === 'unknown') {
|
||||
throw new Error('Could not extract a valid repository name from URL');
|
||||
}
|
||||
return safeName;
|
||||
}
|
||||
|
||||
/** Get the clone target directory for a repo name. */
|
||||
export function getCloneDir(repoName: string): string {
|
||||
// Re-validate at the boundary even though extractRepoName already checked —
|
||||
|
|
@ -247,9 +269,20 @@ export interface CloneOrPullOptions {
|
|||
allowedCloneRoot?: string;
|
||||
expectedRepoName?: string;
|
||||
quarantineRoot?: string;
|
||||
allowAutoSyncSsh?: boolean;
|
||||
timeoutMs?: number;
|
||||
branch?: string;
|
||||
runGitForTest?: typeof runGit;
|
||||
}
|
||||
|
||||
type RunGitOptions = {
|
||||
token?: string;
|
||||
url?: string;
|
||||
timeoutMs?: number;
|
||||
timeoutKillGraceMs?: number;
|
||||
spawnForTest?: typeof spawn;
|
||||
};
|
||||
|
||||
/**
|
||||
* Build the `git clone` argument list for a given URL and target directory.
|
||||
*
|
||||
|
|
@ -319,6 +352,10 @@ export function buildCloneArgs(url: string, targetDir: string): string[] {
|
|||
return ['clone', '--depth', '1', '--', url, targetDir];
|
||||
}
|
||||
|
||||
export function buildBranchCloneArgs(url: string, targetDir: string, branch: string): string[] {
|
||||
return ['clone', '--depth', '1', '--branch', branch, '--', url, targetDir];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a git URL into a comparable form.
|
||||
*
|
||||
|
|
@ -486,7 +523,8 @@ export async function cloneOrPull(
|
|||
// Always validate the requested URL — the prior shape only ran this in
|
||||
// the code path where the repo was cloned. Now it runs unconditionally,
|
||||
// preventing SSRF / blocked-host bypasses even when targetDir already exists.
|
||||
validateGitUrl(url);
|
||||
if (options?.allowAutoSyncSsh) validateAutoSyncRemoteUrl(url);
|
||||
else validateGitUrl(url);
|
||||
await fs.mkdir(cloneRoot, { recursive: true });
|
||||
if (options?.allowedCloneRoot) {
|
||||
await assertDirectoryOwnerAndPermissions(cloneRoot);
|
||||
|
|
@ -511,7 +549,24 @@ export async function cloneOrPull(
|
|||
// whatever remote the dir was originally cloned from.
|
||||
await assertRemoteMatchesRequestedUrl(safeTarget, url);
|
||||
onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' });
|
||||
await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url });
|
||||
const runGitImpl = options?.runGitForTest ?? runGit;
|
||||
if (options?.branch) {
|
||||
await runGitImpl(['fetch', '--depth', '1', 'origin', options.branch], safeTarget, {
|
||||
token: options?.token,
|
||||
url,
|
||||
timeoutMs: options?.timeoutMs,
|
||||
});
|
||||
await runGitImpl(['checkout', options.branch], safeTarget, {
|
||||
token: options?.token,
|
||||
url,
|
||||
timeoutMs: options?.timeoutMs,
|
||||
});
|
||||
}
|
||||
await runGitImpl(['pull', '--ff-only'], safeTarget, {
|
||||
token: options?.token,
|
||||
url,
|
||||
timeoutMs: options?.timeoutMs,
|
||||
});
|
||||
} else {
|
||||
if (targetExists) {
|
||||
throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`);
|
||||
|
|
@ -522,7 +577,14 @@ export async function cloneOrPull(
|
|||
onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` });
|
||||
try {
|
||||
const runGitImpl = options?.runGitForTest ?? runGit;
|
||||
await runGitImpl(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url });
|
||||
const cloneArgs = options?.branch
|
||||
? buildBranchCloneArgs(url, safeTarget, options.branch)
|
||||
: buildCloneArgs(url, safeTarget);
|
||||
await runGitImpl(cloneArgs, undefined, {
|
||||
token: options?.token,
|
||||
url,
|
||||
timeoutMs: options?.timeoutMs,
|
||||
});
|
||||
await assertPostRealpathContainment(cloneRoot, safeTarget);
|
||||
} catch (err: unknown) {
|
||||
if (options?.quarantineRoot) {
|
||||
|
|
@ -731,10 +793,11 @@ export function buildGitEnv(
|
|||
function runGit(
|
||||
args: string[],
|
||||
cwd?: string,
|
||||
options?: { token?: string; url?: string },
|
||||
options?: RunGitOptions,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn('git', args, {
|
||||
const spawnGit = options?.spawnForTest ?? spawn;
|
||||
const proc = spawnGit('git', args, {
|
||||
cwd,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
|
|
@ -742,21 +805,47 @@ function runGit(
|
|||
});
|
||||
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
let timedOut = false;
|
||||
let killTimer: NodeJS.Timeout | undefined;
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
fn();
|
||||
};
|
||||
const timer =
|
||||
options?.timeoutMs && options.timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
proc.kill('SIGTERM');
|
||||
killTimer = setTimeout(() => {
|
||||
proc.kill('SIGKILL');
|
||||
}, options.timeoutKillGraceMs ?? 1_000);
|
||||
}, options.timeoutMs)
|
||||
: undefined;
|
||||
proc.stderr.on('data', (chunk: Buffer) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
if (timedOut) {
|
||||
finish(() => reject(new Error(`git ${args[0]} timed out after ${options?.timeoutMs}ms`)));
|
||||
return;
|
||||
}
|
||||
if (code === 0) finish(resolve);
|
||||
else {
|
||||
// Log full stderr internally but don't expose it to API callers (SSRF mitigation)
|
||||
if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`);
|
||||
reject(new Error(`git ${args[0]} failed (exit code ${code})`));
|
||||
finish(() => reject(new Error(`git ${args[0]} failed (exit code ${code})`)));
|
||||
}
|
||||
});
|
||||
|
||||
proc.on('error', (err) => {
|
||||
reject(new Error(`Failed to spawn git: ${err.message}`));
|
||||
finish(() => reject(new Error(`Failed to spawn git: ${err.message}`)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const runGitForTest = runGit;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5,20 +5,25 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||
|
||||
import {
|
||||
extractRepoNameFromRemoteUrl,
|
||||
getAutoSyncStatePath,
|
||||
getAutoSyncWatchDir,
|
||||
getProjectCommitInfoPath,
|
||||
loadAutoSyncConfig,
|
||||
parseAutoSyncFlag,
|
||||
parseBranchCandidates,
|
||||
parseDurationMs,
|
||||
resolveConfiguredCloneRoot,
|
||||
loadAutoSyncState,
|
||||
saveAutoSyncState,
|
||||
shouldAnalyzeCommit,
|
||||
validateAutoSyncRemoteUrl,
|
||||
validateAutoSyncBranchName,
|
||||
writeProjectCommitInfo,
|
||||
} from '../../src/core/auto-sync/index.js';
|
||||
|
||||
describe('auto-sync', () => {
|
||||
let tempDir: string;
|
||||
let gitnexusHome: string;
|
||||
let oldHome: string | undefined;
|
||||
let oldFlag: string | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
const base = path.join(process.cwd(), '.tmp-test');
|
||||
|
|
@ -27,43 +32,38 @@ describe('auto-sync', () => {
|
|||
gitnexusHome = path.join(tempDir, '.gitnexus');
|
||||
await fs.mkdir(gitnexusHome);
|
||||
oldHome = process.env.GITNEXUS_HOME;
|
||||
oldFlag = process.env.AUTO_UPDATE_AND_ANALYZE_FLAG;
|
||||
process.env.GITNEXUS_HOME = gitnexusHome;
|
||||
delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (oldHome === undefined) delete process.env.GITNEXUS_HOME;
|
||||
else process.env.GITNEXUS_HOME = oldHome;
|
||||
if (oldFlag === undefined) delete process.env.AUTO_UPDATE_AND_ANALYZE_FLAG;
|
||||
else process.env.AUTO_UPDATE_AND_ANALYZE_FLAG = oldFlag;
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('keeps auto sync disabled when the flag is unset or 0', () => {
|
||||
expect(parseAutoSyncFlag(undefined)).toEqual({ enabled: false, reason: 'unset' });
|
||||
expect(parseAutoSyncFlag('0')).toEqual({ enabled: false, reason: 'disabled' });
|
||||
it('places watch runtime artifacts under the watch directory by default', () => {
|
||||
expect(getAutoSyncWatchDir(gitnexusHome)).toBe(path.join(gitnexusHome, 'watch'));
|
||||
expect(getAutoSyncStatePath(gitnexusHome)).toBe(
|
||||
path.join(gitnexusHome, 'watch', 'auto-sync-state.json'),
|
||||
);
|
||||
expect(getProjectCommitInfoPath(gitnexusHome)).toBe(
|
||||
path.join(gitnexusHome, 'watch', 'project_commit_info.txt'),
|
||||
);
|
||||
});
|
||||
|
||||
it('enables auto sync only for the explicit value 1', () => {
|
||||
expect(parseAutoSyncFlag('1')).toEqual({ enabled: true });
|
||||
expect(parseAutoSyncFlag('true')).toEqual({
|
||||
enabled: false,
|
||||
reason: 'invalid',
|
||||
message: '[auto-sync] AUTO_UPDATE_AND_ANALYZE_FLAG must be 0 or 1; got "true". Auto sync is disabled.',
|
||||
});
|
||||
});
|
||||
|
||||
it('loads sync_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => {
|
||||
it('loads watch_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(gitnexusHome, 'sync_config.yml'),
|
||||
path.join(gitnexusHome, 'watch_config.yml'),
|
||||
[
|
||||
'sync_interval_minutes: 10',
|
||||
'max_concurrency: 3',
|
||||
'repo_git_timeout: 12s',
|
||||
'analyze_failure_threshold: 2',
|
||||
'projects:',
|
||||
' - local_path: /tmp/repos',
|
||||
' gitnexus_group: back_end',
|
||||
' branch: test, master, test',
|
||||
' group_name: back_end',
|
||||
' branches: [test, master, test]',
|
||||
' remote_urls:',
|
||||
' - git@gitee.com:qts_server/qts_account.git',
|
||||
].join('\n'),
|
||||
|
|
@ -73,49 +73,97 @@ describe('auto-sync', () => {
|
|||
|
||||
expect(loaded.ok).toBe(true);
|
||||
if (!loaded.ok) throw new Error('expected config to load');
|
||||
expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'sync_config.yml'));
|
||||
expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'watch_config.yml'));
|
||||
expect(loaded.config.syncIntervalMinutes).toBe(10);
|
||||
expect(loaded.config.maxConcurrency).toBe(3);
|
||||
expect(loaded.config.repoGitTimeoutMs).toBe(12_000);
|
||||
expect(loaded.config.analyzeFailureThreshold).toBe(2);
|
||||
expect(loaded.config.projects[0]).toMatchObject({
|
||||
localPath: '/tmp/repos',
|
||||
gitnexusGroup: 'back_end',
|
||||
groupName: 'back_end',
|
||||
branches: ['test', 'master'],
|
||||
remoteUrls: ['git@gitee.com:qts_server/qts_account.git'],
|
||||
});
|
||||
});
|
||||
|
||||
it('defaults repo_git_timeout and max_concurrency and allows empty group_name', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(gitnexusHome, 'watch_config.yml'),
|
||||
[
|
||||
'sync_interval_minutes: 10',
|
||||
'projects:',
|
||||
' - local_path: /tmp/repos',
|
||||
' group_name: ""',
|
||||
' branch: master',
|
||||
' remote_urls:',
|
||||
' - git@github.com:owner/repo.git',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
|
||||
expect(loaded.ok).toBe(true);
|
||||
if (!loaded.ok) throw new Error('expected config');
|
||||
expect(loaded.config.repoGitTimeoutMs).toBe(10_000);
|
||||
expect(loaded.config.maxConcurrency).toBe(1);
|
||||
expect(loaded.config.analyzeFailureThreshold).toBe(3);
|
||||
expect(loaded.config.projects[0].groupName).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects invalid analyze_failure_threshold values', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(gitnexusHome, 'watch_config.yml'),
|
||||
[
|
||||
'sync_interval_minutes: 10',
|
||||
'analyze_failure_threshold: 1',
|
||||
'projects:',
|
||||
' - local_path: /tmp/repos',
|
||||
' branch: master',
|
||||
' remote_urls:',
|
||||
' - git@github.com:owner/repo.git',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
|
||||
expect(loaded.ok).toBe(false);
|
||||
if (loaded.ok) throw new Error('expected invalid config');
|
||||
expect(loaded.message).toContain('analyze_failure_threshold must be an integer >= 2');
|
||||
});
|
||||
|
||||
it('reports missing config without throwing', async () => {
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
|
||||
expect(loaded).toEqual({
|
||||
ok: false,
|
||||
reason: 'missing',
|
||||
message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'sync_config.yml')}. Auto sync is skipped.`,
|
||||
message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'watch_config.yml')}. Auto sync is skipped.`,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports invalid config without throwing', async () => {
|
||||
await fs.writeFile(path.join(gitnexusHome, 'sync_config.yml'), 'projects: []\n');
|
||||
await fs.writeFile(path.join(gitnexusHome, 'watch_config.yml'), 'projects: []\n');
|
||||
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
|
||||
expect(loaded.ok).toBe(false);
|
||||
if (loaded.ok) throw new Error('expected invalid config');
|
||||
expect(loaded.reason).toBe('invalid');
|
||||
expect(loaded.message).toContain('[auto-sync] Invalid sync_config.yml:');
|
||||
expect(loaded.message).toContain('[auto-sync] Invalid watch_config.yml:');
|
||||
expect(loaded.message).toContain('sync_interval_minutes must be a positive integer');
|
||||
expect(loaded.message).toContain('projects must contain at least one project');
|
||||
});
|
||||
|
||||
it('rejects missing, relative, and traversal local_path values at config load', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(gitnexusHome, 'sync_config.yml'),
|
||||
path.join(gitnexusHome, 'watch_config.yml'),
|
||||
[
|
||||
'sync_interval_minutes: 10',
|
||||
'projects:',
|
||||
' - local_path: ../repos',
|
||||
' branch: master',
|
||||
' remote_urls:',
|
||||
' - https://example.com/team/repo.git',
|
||||
' - git@github.com:team/repo.git',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
|
|
@ -147,6 +195,18 @@ describe('auto-sync', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('allows the default GitNexus repos directory as an auto-sync clone root', async () => {
|
||||
const root = path.join(gitnexusHome, 'repos');
|
||||
await fs.mkdir(root, { recursive: true });
|
||||
|
||||
await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
root,
|
||||
quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects symlinks in configured clone root paths', async () => {
|
||||
const realRoot = path.join(tempDir, 'real-root');
|
||||
const linkRoot = path.join(tempDir, 'link-root');
|
||||
|
|
@ -163,33 +223,88 @@ describe('auto-sync', () => {
|
|||
await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
root,
|
||||
quarantineRoot: path.join(gitnexusHome, 'quarantine'),
|
||||
quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'),
|
||||
quarantineRetentionDays: 14,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('creates missing configured clone roots before watch clone work', async () => {
|
||||
const root = path.join(tempDir, 'missing-repos');
|
||||
|
||||
await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
root,
|
||||
quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'),
|
||||
}),
|
||||
);
|
||||
expect((await fs.stat(root)).isDirectory()).toBe(true);
|
||||
});
|
||||
|
||||
it('parses branch strings and arrays with trimming and de-duplication', () => {
|
||||
expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']);
|
||||
expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']);
|
||||
});
|
||||
|
||||
it('rejects unsafe auto-sync branch names', () => {
|
||||
expect(() => validateAutoSyncBranchName('feature/good-branch')).not.toThrow();
|
||||
expect(() => validateAutoSyncBranchName('-upload-pack=evil')).toThrow('must not start');
|
||||
expect(() => validateAutoSyncBranchName('feature bad')).toThrow('whitespace');
|
||||
expect(() => validateAutoSyncBranchName('feature..bad')).toThrow('must not contain ".."');
|
||||
expect(() => validateAutoSyncBranchName('bad:ref')).toThrow('not allowed');
|
||||
});
|
||||
|
||||
it('extracts safe repository names from remote URLs', () => {
|
||||
expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe(
|
||||
'qts_account',
|
||||
);
|
||||
expect(extractRepoNameFromRemoteUrl('https://example.com/team/repo-name.git')).toBe(
|
||||
'repo-name',
|
||||
);
|
||||
expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe('repo-name');
|
||||
});
|
||||
|
||||
it('rejects unsafe repository names without sanitizing them', () => {
|
||||
expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/repo$name.git')).toThrow(
|
||||
expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/repo$name.git')).toThrow(
|
||||
'valid repository name',
|
||||
);
|
||||
expect(() => extractRepoNameFromRemoteUrl('https://example.com/team/..')).toThrow(
|
||||
'valid repository name',
|
||||
expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/..')).toThrow('traversal');
|
||||
});
|
||||
|
||||
it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => {
|
||||
expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow();
|
||||
expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow();
|
||||
expect(() => validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git')).not.toThrow();
|
||||
expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow('must use');
|
||||
expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow('must use');
|
||||
expect(() => validateAutoSyncRemoteUrl('user@github.com:owner/repo.git')).toThrow('must use');
|
||||
expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow('host must be');
|
||||
});
|
||||
|
||||
it('parses repo git timeout durations', () => {
|
||||
expect(parseDurationMs('10s')).toBe(10_000);
|
||||
expect(parseDurationMs('2m')).toBe(120_000);
|
||||
expect(parseDurationMs('5000ms')).toBe(5000);
|
||||
expect(parseDurationMs('10')).toBe(10_000);
|
||||
expect(parseDurationMs(10)).toBe(10_000);
|
||||
});
|
||||
|
||||
it('keeps branch compatibility but rejects branch and branches together', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(gitnexusHome, 'watch_config.yml'),
|
||||
[
|
||||
'sync_interval_minutes: 10',
|
||||
'projects:',
|
||||
' - local_path: /tmp/repos',
|
||||
' branch: master',
|
||||
' branches: [develop]',
|
||||
' remote_urls:',
|
||||
' - git@github.com:owner/repo.git',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const loaded = await loadAutoSyncConfig();
|
||||
|
||||
expect(loaded.ok).toBe(false);
|
||||
if (loaded.ok) throw new Error('expected invalid config');
|
||||
expect(loaded.message).toContain('must not set both branch and branches');
|
||||
});
|
||||
|
||||
it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => {
|
||||
|
|
@ -217,6 +332,8 @@ describe('auto-sync', () => {
|
|||
codeCommitId: 'abc',
|
||||
analyzedCommitId: 'abc',
|
||||
lastAnalyzeStatus: 'success',
|
||||
analyzeConsecutiveFailures: 2,
|
||||
lastAnalyzeError: 'old error',
|
||||
lastSyncTime: '2026-06-30T00:00:00.000Z',
|
||||
},
|
||||
},
|
||||
|
|
@ -228,11 +345,13 @@ describe('auto-sync', () => {
|
|||
);
|
||||
await expect(loadAutoSyncState(statePath)).resolves.toEqual({
|
||||
'/tmp/repos/qts_account|master': {
|
||||
codeCommitId: 'abc',
|
||||
analyzedCommitId: 'abc',
|
||||
lastAnalyzeStatus: 'success',
|
||||
lastSyncTime: '2026-06-30T00:00:00.000Z',
|
||||
},
|
||||
codeCommitId: 'abc',
|
||||
analyzedCommitId: 'abc',
|
||||
lastAnalyzeStatus: 'success',
|
||||
analyzeConsecutiveFailures: 2,
|
||||
lastAnalyzeError: 'old error',
|
||||
lastSyncTime: '2026-06-30T00:00:00.000Z',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -247,4 +366,48 @@ describe('auto-sync', () => {
|
|||
`[auto-sync] Ignoring unreadable or corrupt state file: ${statePath}. State will be rebuilt.\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it('writes project_commit_info.txt atomically', async () => {
|
||||
const infoPath = path.join(tempDir, 'project_commit_info.txt');
|
||||
|
||||
await writeProjectCommitInfo(
|
||||
[
|
||||
{
|
||||
remoteUrl: 'git@github.com:owner/repo.git',
|
||||
localPath: '/tmp/repos/repo',
|
||||
branch: 'master',
|
||||
codeCommitId: 'abc',
|
||||
analyzedCommitId: 'abc',
|
||||
status: 'success',
|
||||
analyzeConsecutiveFailures: 0,
|
||||
analyzeFailureThreshold: 3,
|
||||
lastSyncTime: '2026-06-30T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
remoteUrl: 'git@github.com:owner/bad.git',
|
||||
localPath: '/tmp/repos/bad',
|
||||
branch: 'master',
|
||||
codeCommitId: 'def',
|
||||
analyzedCommitId: 'abc',
|
||||
status: 'threshold_skipped',
|
||||
analyzeConsecutiveFailures: 3,
|
||||
analyzeFailureThreshold: 3,
|
||||
lastAnalyzeError: 'parser crashed',
|
||||
lastSyncTime: '2026-06-30T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
infoPath,
|
||||
);
|
||||
|
||||
const content = await fs.readFile(infoPath, 'utf-8');
|
||||
expect(content).toContain('remote: git@github.com:owner/repo.git');
|
||||
expect(content).toContain('code_commit: abc');
|
||||
expect(content).toContain('analyze_consecutive_failures: 0');
|
||||
expect(content).toContain('analyze_failure_threshold: 3');
|
||||
expect(content).toContain('status: threshold_skipped');
|
||||
expect(content).toContain('last_analyze_error: parser crashed');
|
||||
await expect(fs.readdir(tempDir)).resolves.not.toContain(
|
||||
expect.stringContaining('project_commit_info.txt.tmp'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@ function runHelp(command: string, env: NodeJS.ProcessEnv = {}) {
|
|||
}
|
||||
|
||||
function runHelpArgs(args: string[], env: NodeJS.ProcessEnv = {}) {
|
||||
return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args, '--help'], {
|
||||
return runCliArgs([...args, '--help'], env);
|
||||
}
|
||||
|
||||
function runCliArgs(args: string[], env: NodeJS.ProcessEnv = {}) {
|
||||
return spawnSync(process.execPath, ['--import', 'tsx', cliEntry, ...args], {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, ...env },
|
||||
|
|
@ -236,6 +240,42 @@ describe('CLI help surface', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('watch help exposes lifecycle actions and state files', () => {
|
||||
const result = runHelp('watch');
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('gitnexus watch [options] [action]');
|
||||
expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status');
|
||||
expect(result.stdout).toContain('GITNEXUS_HOME/watch_config.yml');
|
||||
expect(result.stdout).toContain('GITNEXUS_HOME/watch/watch.pid');
|
||||
expect(result.stdout).toContain('GITNEXUS_HOME/watch/project_commit_info.txt');
|
||||
});
|
||||
|
||||
it('watch init creates the default watch_config.yml and does not overwrite it', () => {
|
||||
const home = fs.mkdtempSync(path.join(repoRoot, '.tmp-test/gitnexus-watch-init-'));
|
||||
try {
|
||||
const first = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home });
|
||||
const configPath = path.join(home, 'watch_config.yml');
|
||||
|
||||
expect(first.status).toBe(0);
|
||||
expect(first.stdout).toContain(`Created ${configPath}`);
|
||||
const config = fs.readFileSync(configPath, 'utf8');
|
||||
expect(config).toContain('sync_interval_minutes: 10');
|
||||
expect(config).toContain('analyze_failure_threshold: 3');
|
||||
expect(config).toContain(`local_path: ${path.join(home, 'repo')}`);
|
||||
expect(config).not.toContain('/abs/path/to/repos');
|
||||
expect(config).toContain('git@github.com:owner/repo.git');
|
||||
|
||||
const second = runCliArgs(['watch', 'init'], { GITNEXUS_HOME: home });
|
||||
|
||||
expect(second.status).toBe(1);
|
||||
expect(second.stderr).toContain(`Config already exists: ${configPath}`);
|
||||
expect(fs.readFileSync(configPath, 'utf8')).toBe(config);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('wiki help shows provider, review, and verbose flags', () => {
|
||||
const result = runHelp('wiki');
|
||||
|
||||
|
|
|
|||
|
|
@ -16,20 +16,24 @@ vi.mock('../../src/core/logger.js', () => ({
|
|||
|
||||
import {
|
||||
extractRepoName,
|
||||
extractWebRepoName,
|
||||
getCloneDir,
|
||||
validateGitUrl,
|
||||
cloneOrPull,
|
||||
buildCloneArgs,
|
||||
buildBranchCloneArgs,
|
||||
buildGitEnv,
|
||||
normalizeGitUrlForCompare,
|
||||
assertRemoteMatchesRequestedUrl,
|
||||
isAzureDevOpsUrl,
|
||||
warnIfInsecureAzureConfig,
|
||||
runGitForTest,
|
||||
} from '../../src/server/git-clone.js';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import fs from 'node:fs/promises';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { getRemoteOriginUrl } from '../../src/storage/git.js';
|
||||
import { getGlobalDir } from '../../src/storage/repo-manager.js';
|
||||
|
||||
|
|
@ -365,6 +369,20 @@ describe('git-clone', () => {
|
|||
expect(args.some((a) => a.toLowerCase().includes('authorization'))).toBe(false);
|
||||
expect(args.some((a) => a.includes('extraHeader'))).toBe(false);
|
||||
});
|
||||
|
||||
it('adds --branch before the URL separator for branch-specific clones', () => {
|
||||
const args = buildBranchCloneArgs('git@github.com:owner/repo.git', '/safe/target', 'develop');
|
||||
expect(args).toEqual([
|
||||
'clone',
|
||||
'--depth',
|
||||
'1',
|
||||
'--branch',
|
||||
'develop',
|
||||
'--',
|
||||
'git@github.com:owner/repo.git',
|
||||
'/safe/target',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildGitEnv — token injection', () => {
|
||||
|
|
@ -551,6 +569,57 @@ describe('git-clone', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('keeps regular cloneOrPull restricted to http and https URLs', async () => {
|
||||
const root = await mkControlledRoot('gitnexus-controlled-root-');
|
||||
try {
|
||||
await expect(
|
||||
cloneOrPull('git@github.com:owner/repo.git', path.join(root, 'repo'), undefined, {
|
||||
allowedCloneRoot: root,
|
||||
expectedRepoName: 'repo',
|
||||
}),
|
||||
).rejects.toThrow('Invalid URL');
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('allows auto-sync SSH SCP clone URLs with a per-repo timeout', async () => {
|
||||
const root = await mkControlledRoot('gitnexus-controlled-root-');
|
||||
const target = path.join(root, 'repo');
|
||||
const runGitForTest = vi.fn(async () => {
|
||||
await fs.mkdir(target);
|
||||
});
|
||||
try {
|
||||
await expect(
|
||||
cloneOrPull('git@gitlab.com:group/subgroup/repo.git', target, undefined, {
|
||||
allowedCloneRoot: root,
|
||||
expectedRepoName: 'repo',
|
||||
allowAutoSyncSsh: true,
|
||||
timeoutMs: 10_000,
|
||||
branch: 'develop',
|
||||
runGitForTest,
|
||||
}),
|
||||
).resolves.toBe(target);
|
||||
|
||||
expect(runGitForTest).toHaveBeenCalledWith(
|
||||
[
|
||||
'clone',
|
||||
'--depth',
|
||||
'1',
|
||||
'--branch',
|
||||
'develop',
|
||||
'--',
|
||||
'git@gitlab.com:group/subgroup/repo.git',
|
||||
target,
|
||||
],
|
||||
undefined,
|
||||
{ token: undefined, url: 'git@gitlab.com:group/subgroup/repo.git', timeoutMs: 10_000 },
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('allows an explicitly controlled auto-sync clone root outside the default root', async () => {
|
||||
const root = await mkControlledRoot('gitnexus-controlled-root-');
|
||||
try {
|
||||
|
|
@ -810,6 +879,30 @@ describe('git-clone', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('extractWebRepoName — API clone compatibility', () => {
|
||||
it('sanitizes repo names with spaces and unsafe directory characters at the web boundary', () => {
|
||||
expect(
|
||||
extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces'),
|
||||
).toBe('My_Repo_With_Spaces');
|
||||
expect(extractWebRepoName('https://example.com/team/repo$name.git')).toBe('repo_name');
|
||||
});
|
||||
|
||||
it('keeps Windows reserved names from becoming clone directories', () => {
|
||||
expect(() => extractWebRepoName('https://example.com/team/CON.git')).toThrow(
|
||||
'valid repository name',
|
||||
);
|
||||
expect(() => extractWebRepoName('https://example.com/team/NUL.txt')).toThrow(
|
||||
'valid repository name',
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves strict extractRepoName behavior unchanged for internal callers', () => {
|
||||
expect(() => extractRepoName('https://example.com/team/repo$name.git')).toThrow(
|
||||
'valid repository name',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateGitUrl — Azure DevOps URLs', () => {
|
||||
it('allows self-hosted Azure DevOps Server URLs', () => {
|
||||
expect(() =>
|
||||
|
|
@ -988,4 +1081,41 @@ describe('git-clone', () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('runGit timeout', () => {
|
||||
it('waits for close and sends SIGKILL after the grace period before returning timeout', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
kill: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.kill = vi.fn();
|
||||
const spawnForTest = vi.fn(() => child) as unknown as typeof spawn;
|
||||
|
||||
const promise = runGitForTest(['clone'], undefined, {
|
||||
timeoutMs: 20,
|
||||
timeoutKillGraceMs: 20,
|
||||
spawnForTest,
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
let settled = false;
|
||||
promise.catch(() => {}).finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
await vi.runAllTicks();
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGTERM');
|
||||
expect(settled).toBe(false);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL');
|
||||
child.emit('close', null);
|
||||
await expect(promise).rejects.toThrow('timed out after 20ms');
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue