From 5e6b79deba67bc521a58350d586dcb6ea89c2f9d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:59:53 -0500 Subject: [PATCH 01/21] fix(cli): do not call zero-symbol detect-changes a clean tree (#3138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): do not call zero-symbol detect-changes a clean tree Print backend summary.message and distinguish a parsed diff with no overlapping indexed symbols from an empty git diff. Pin --color=never so color.ui=always cannot hide +++ b/ headers. * fix(cli): localize clean-tree detect-changes and skip no-overlap when partial Production empty diffs carry English summary.message; route that through t() so zh-CN fires. Do not claim no indexed-symbol overlap on queryDegraded partial results. Pin formatter tests to en and cover the production payload shapes. * fix(cli): prettier detect-changes-format and degraded eval assertion --------- Co-authored-by: Gergő Magyar --- gitnexus/src/cli/detect-changes-format.ts | 24 +++++ gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 2 + gitnexus/src/mcp/local/local-backend.ts | 3 + gitnexus/test/unit/detect-changes-eol.test.ts | 3 +- .../test/unit/detect-changes-format.test.ts | 92 +++++++++++++++++++ gitnexus/test/unit/eval-formatters.test.ts | 3 +- 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 gitnexus/test/unit/detect-changes-format.test.ts diff --git a/gitnexus/src/cli/detect-changes-format.ts b/gitnexus/src/cli/detect-changes-format.ts index 98ecffa3e..9e91cc330 100644 --- a/gitnexus/src/cli/detect-changes-format.ts +++ b/gitnexus/src/cli/detect-changes-format.ts @@ -6,6 +6,7 @@ type DetectChangesSummary = { changed_count?: number; affected_count?: number; risk_level?: string; + message?: string; }; type ChangedSymbol = { @@ -55,6 +56,29 @@ export function formatDetectChangesResult(result: unknown): string { ); if ((summary.changed_count ?? 0) === 0) { + // Parse-fail payloads set `partial` and an honest `message` (#2915/#3131). + // Production *clean* trees also set English `message: 'No changes detected.'` + // — that must go through `t('tool.detectChanges.noChanges')` or zh-CN never + // fires. Only pass the backend string through on a degraded/parse-fail run. + if ( + payload.partial && + typeof summary.message === 'string' && + summary.message.trim().length > 0 + ) { + return [...notes, summary.message.trim()].join('\n'); + } + // Confirmed no-overlap: files parsed, mapping succeeded, zero symbols. + // `queryDegraded` is `partial: true` with the same counts and no message — + // do not call that a confirmed mapping (#3131 honesty). + if (!payload.partial && (summary.changed_files ?? 0) > 0) { + return [ + ...notes, + t('tool.detectChanges.noOverlappingSymbols', { files: summary.changed_files }), + ].join('\n'); + } + if (payload.partial) { + return notes.join('\n'); + } return [...notes, t('tool.detectChanges.noChanges')].join('\n'); } diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index aba8e78f0..c54f5707b 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -76,6 +76,8 @@ export const en = { 'tool.warn.unknownKind': "--kind '{{kind}}' is not a known symbol kind (e.g. Function, Class, Method); it will not narrow the result.", 'tool.detectChanges.noChanges': 'No changes detected.', + 'tool.detectChanges.noOverlappingSymbols': + 'Diff touched {{files}} file(s) but no indexed symbols overlap those hunks — not a clean tree.', 'tool.detectChanges.partial': 'PARTIAL RESULT: a graph query failed, so changed symbols may be missing. Do not read this as a clean pre-commit check.', 'tool.detectChanges.truncated': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 5f3b52040..a8001af8a 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -78,6 +78,8 @@ export const zhCN = { 'tool.warn.unknownKind': "--kind '{{kind}}' 不是已知的符号类型（如 Function、Class、Method），不会用于缩小结果范围。", 'tool.detectChanges.noChanges': '未检测到变更。', + 'tool.detectChanges.noOverlappingSymbols': + 'diff 触及 {{files}} 个文件，但没有索引符号与这些 hunk 重叠 — 并非干净工作区。', 'tool.detectChanges.partial': '结果不完整：图查询失败，可能遗漏已变更符号。请勿将其视为通过的提交前检查。', 'tool.detectChanges.truncated': diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index a5a7b85ce..783cb1255 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -988,6 +988,9 @@ export function buildDetectChangesDiffArgs(scope: string, baseRef?: string): str 'diff', '--ignore-cr-at-eol', '--no-ext-diff', + // color.ui=always prefixes `+++ b/` with ANSI, so parseDiffHunks sees zero + // files and the CLI used to print a clean "No changes detected." (#3131). + '--color=never', '--src-prefix=a/', '--dst-prefix=b/', ]; diff --git a/gitnexus/test/unit/detect-changes-eol.test.ts b/gitnexus/test/unit/detect-changes-eol.test.ts index 9ed43a9b4..bb3da5328 100644 --- a/gitnexus/test/unit/detect-changes-eol.test.ts +++ b/gitnexus/test/unit/detect-changes-eol.test.ts @@ -8,11 +8,12 @@ import { parseDiffHunks } from '../../src/storage/git.js'; import { diffArgsFor } from '../helpers/detect-changes-diff-args.js'; import { commitAll, initGitRepo } from '../helpers/temp-git-repo.js'; -/** The five flags every scope carries, ahead of its own ref/staging arguments. */ +/** The six flags every scope carries, ahead of its own ref/staging arguments. */ const GUARD_FLAGS = [ 'diff', '--ignore-cr-at-eol', '--no-ext-diff', + '--color=never', '--src-prefix=a/', '--dst-prefix=b/', ]; diff --git a/gitnexus/test/unit/detect-changes-format.test.ts b/gitnexus/test/unit/detect-changes-format.test.ts new file mode 100644 index 000000000..18f21eeeb --- /dev/null +++ b/gitnexus/test/unit/detect-changes-format.test.ts @@ -0,0 +1,92 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { formatDetectChangesResult } from '../../src/cli/detect-changes-format.js'; +import { setCliLanguage } from '../../src/cli/i18n/index.js'; + +describe('formatDetectChangesResult — zero-symbol honesty (#3131)', () => { + beforeEach(() => { + setCliLanguage('en'); + }); + + afterEach(() => { + setCliLanguage(null); + }); + + it('prints backend parse-fail message instead of a generic all-clear', () => { + const text = formatDetectChangesResult({ + partial: true, + summary: { + changed_count: 0, + affected_count: 0, + risk_level: 'unknown', + message: 'Could not parse the git diff output — no file headers recognised.', + }, + }); + expect(text).toContain('PARTIAL RESULT'); + expect(text).toContain('Could not parse the git diff output'); + expect(text).not.toContain('No changes detected.'); + }); + + it('does not call a parsed diff with no symbol overlap a clean tree', () => { + const text = formatDetectChangesResult({ + summary: { + changed_count: 0, + affected_count: 0, + changed_files: 1, + risk_level: 'low', + }, + }); + expect(text).toMatch(/Diff touched 1 file/); + expect(text).not.toContain('No changes detected.'); + expect(text).not.toContain('PARTIAL RESULT'); + }); + + it('does not claim no-overlap when a degraded query left changed_count at zero', () => { + const text = formatDetectChangesResult({ + partial: true, + summary: { + changed_count: 0, + affected_count: 0, + changed_files: 1, + risk_level: 'unknown', + }, + }); + expect(text).toContain('PARTIAL RESULT'); + expect(text).not.toMatch(/no indexed symbols overlap/i); + expect(text).not.toContain('No changes detected.'); + }); + + it('keeps the clean-tree sentence only when git produced no files', () => { + const text = formatDetectChangesResult({ + summary: { changed_count: 0, affected_count: 0, changed_files: 0, risk_level: 'none' }, + }); + expect(text).toBe('No changes detected.'); + }); + + it('localizes the production clean-tree payload that carries English summary.message', () => { + setCliLanguage('zh-CN'); + const text = formatDetectChangesResult({ + summary: { + changed_count: 0, + affected_count: 0, + risk_level: 'none', + message: 'No changes detected.', + }, + }); + expect(text).toBe('未检测到变更。'); + }); + + it('localizes confirmed no-overlap under GITNEXUS_LANG=zh-CN', () => { + setCliLanguage('zh-CN'); + const text = formatDetectChangesResult({ + summary: { + changed_count: 0, + affected_count: 0, + changed_files: 1, + risk_level: 'low', + }, + }); + expect(text).toContain('diff 触及 1 个文件'); + expect(text).not.toContain('未检测到变更。'); + expect(text).not.toContain('No changes detected.'); + }); +}); diff --git a/gitnexus/test/unit/eval-formatters.test.ts b/gitnexus/test/unit/eval-formatters.test.ts index cdd12b4fd..57cff0b81 100644 --- a/gitnexus/test/unit/eval-formatters.test.ts +++ b/gitnexus/test/unit/eval-formatters.test.ts @@ -604,7 +604,8 @@ describe('formatDetectChangesResult', () => { // counts at zero. Without the note the pre-commit gate reads as "clean". const result = formatDetectChangesResult({ partial: true, summary: { changed_count: 0 } }); expect(result).toContain('PARTIAL RESULT'); - expect(result).toContain('No changes detected.'); + expect(result).toContain('a graph query failed'); + expect(result).not.toContain('No changes detected.'); }); it('flags a degraded run that still found symbols', () => { From 3aa62be717a579d4644364d91fdd50c1b8b5c286 Mon Sep 17 00:00:00 2001 From: Yayler Date: Thu, 3 Sep 2026 02:40:18 +0800 Subject: [PATCH 02/21] feat: add gitnexus auto-sync for scheduled remote clone and analyze (#2493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adds an opt-in auto sync and analysis loop for GitNexus * adds an opt-in auto sync and analysis loop for GitNexus,gitnexus watch [init|start|restart|stop|status] * adds an opt-in auto sync and analysis loop for GitNexus,gitnexus watch [init|start|restart|stop|status] * fix: address PR review cleanup * Prettier code style * merge main * fix(watch): protect local repos and cancel active analysis * fix(watch): harden auto-sync lifecycle and locking - validate watch process identity before lifecycle operations\n- serialize registry, analysis, and LadybugDB access with recoverable locks\n- harden clone paths, symlinks, hooks, quarantine, and worker timeouts\n- install procps in the CLI image for reliable Docker watch control\n- add focused regression coverage for lifecycle, locks, clone, and registry behavior * update agents & claude md * merge main * fix(watch): harden auto-sync lifecycle * fix(watch): normalize SSH repo identity paths * fix(watch): normalize SSH repo identity paths * fix(watch): safely cancel analysis across platforms * fix(auto-sync): close worker and group sync failure paths * fix(auto-sync): drop retired allowStale from group sync allowStale was removed from SyncOptions, which broke typecheck and CI on this PR. Co-authored-by: Cursor * fix(watch): satisfy prefer-const and Prettier in auto-sync The watch timers are assigned exactly once, so prefer-const rejected the deferred `let` declarations. They are only read from `stop()` and the control poll, both of which run after the assignments, so binding them at creation is safe and drops the now-dead undefined guards. Remaining files are formatting only. Co-authored-by: Cursor * fix(watch): make lock identity absolute and stop three fail-open paths Lock owner identity was rendered by `ps -o lstart=` through localtime and the active locale, so the same live process produced a different string under a different TZ. A mismatch reads as PID reuse, so one daemon could reclaim a mutex another still held. Pin TZ=UTC and LC_ALL=C. The owner record also carried no hostname, so a holder on another machine was judged by this kernel's view of its PID — always "stale" — and its lock stolen whenever GITNEXUS_HOME is a shared volume. Record and compare the hostname, as the index lock already does. Ownership verification threw unconditionally on win32, which is reached once per project per tick, so watch reported `running` and then failed every repo forever. POSIX uid/mode cannot be checked there; skip those two assertions and keep the dangerous-root, symlink, containment and internal-root guards. Also: quarantine sweep now refuses a symlinked root instead of deleting through it; an unreadable state file propagates instead of being rewritten as empty state, which used to erase every repo's analyzed commit and failure count; a failed staging cleanup no longer strands a published lock with no release handle; and the concurrency runner settles every worker before surfacing a failure so cancellation cannot orphan a live analyze fork. Co-authored-by: Cursor * fix(watch): land the deferred review findings Six findings that were deferred from the review backlog, plus the docs they change. Worker heap: admission allowed `floor(availableMemoryGB / 2)` slots while every fork was handed the whole machine's heap cap, so the budget meant nothing as soon as an operator raised max_concurrency. Divide the cap by the repos actually analyzed in parallel. The default single-project path is unchanged. Registration: the parent registered without a branch, so it always took the primary/flat arm and relabelled a pinned branch entry on the branch-fallback path. Reproduce the worker's own resolveBranchPlacement decision instead. Cancellation: requestCancellation cleared the only timer and settled nothing, so a worker wedged past its safe point left the promise pending forever, wedging activeRun and hanging `watch stop`. Add a 5s grace after which the parent stops waiting and releases the IPC channel's hold on its event loop. The child is still never killed — it may be inside native work. overwrite_local_changes: `checkout --force` rewrites tracked files only, so untracked sources survived and were indexed as if they came from the remote. `git clean -fd -e /.gitnexus` after checkout; no -x/-X, so ignored paths and GitNexus's own storage survive. Quarantine: age alone never bounds a repo that fails every tick, since each partial clone is younger than the retention window. Keep the five newest per repo. Validation: repo_git_timeout is now bounded by the lesser of an hour and the sync interval, which is also the guard for the bare-number-means-seconds slip (`600000` meant ~7 days and cleared the timer ceiling). And the remote URL's final segment is validated at config load rather than failing once per tick inside the sync loop. Co-authored-by: Cursor * fix(watch): release an errored worker, and stop rejecting dotted repo names Three findings from the latest review pass. The 'error' handler settles immediately rather than waiting out the grace, so cleanup() clears the grace timer that would otherwise have released the child. An errored IPC channel does not mean the worker stopped, so release it on that path too — still no kill. The traversal guard tested the raw path for '..', which also rejected an ordinary name like owner/foo..bar that the repository-name rule accepts. Traversal is a whole segment, so test segments. The heap-cap test left two runs and their real timers pending; it now stubs timers and settles both promises. Registration coverage now pins the branch slot rather than leaving it implicit. Co-authored-by: Cursor * fix(watch): validate namespace segments and pin the stopped process identity Replacing the raw-string `..` test with a per-segment one dropped a guard: a segment like `..\..\outside` is not literally `..`, so it passed, and those segments build the clone path — on Windows the backslashes are separators. Hold every namespace segment to the same charset as the repo name, which keeps a separator out of a segment while still allowing an ordinary `foo..bar`. The final segment keeps its own check so a bad repo name keeps its own message. The stop wait polled liveness by pid alone, so a pid reused mid-wait would have it wait on an unrelated process and then report the watch stopped. Compare the process start time recorded for the owner, which also returns sooner. Registration now omits `branch` for a primary index instead of passing it as undefined, so that call keeps the shape it had before this branch. Co-authored-by: Cursor * fix(cli): ship auto-sync as the remote daemon, reserve gitnexus watch. Keep analyze --watch for local incremental re-index and stop the top-level watch verb from starting a clone/pull loop. Co-authored-by: Cursor * fix(auto-sync): reject invalid branch refs and verify status identity (#2493) Reject leading slashes and per-component trailing dots in configured branches, and verify the live watch owner before trusting a stored error status. Co-authored-by: Cursor * fix(auto-sync): reject ownerIds that can escape the watch directory (#2493) Stop interpolating a tampered ownerId into the stop-request filename; only basename-safe values are treated as owners. Co-authored-by: Cursor * fix(auto-sync): recognize auto-sync in the watch-process identity check (#2493) Stop/status were still looking for a standalone watch token after the command rename, so a live gitnexus auto-sync start process would be refused as unrelated. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command * fix(auto-sync): reject boolean max_concurrency instead of coercing it to 1 (#2493) Number(true) is 1, so a YAML boolean would have passed the integer check and silently meant one worker. Co-authored-by: Cursor * fix(auto-sync): swallow status errors in the watch finally path (#2493) An uncaught updateStatus rejection in finally became an unhandled rejection. Skip the clone-root symlink test on Windows, where directory symlinks need privileges. Align the group-lock comment with fail-closed registry timeouts. Co-authored-by: Cursor * fix(auto-sync): catch cancelling status-write failures (#2493) Fire-and-forget updateStatus('cancelling') could become an unhandled rejection, the same class as the finally-path status write. Co-authored-by: Cursor * fix(auto-sync): ignore queued interval ticks after stop (#2493) clearInterval does not cancel a timer callback already queued. Guard runSafely on stopping so shutdown cannot start a new un-cancellable run. Co-authored-by: Cursor * fix(auto-sync): report stored watch status timestamps (#2493) status should show when the watch last entered a state, not when the CLI queried it. The failure-count test still expects 1 after a new commit resets the streak; rename it so that reset is explicit. Co-authored-by: Cursor * style(auto-sync): apply prettier to starter status logger (#2493) Co-authored-by: Cursor --------- Co-authored-by: weiyf Co-authored-by: Gergő Magyar Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .claude/skills/gitnexus-cli/SKILL.md | 2 +- .gitignore | 2 + Dockerfile.cli | 5 +- README.md | 40 + eslint-rules/require-safe-parse.mjs | 6 +- .../skills/gitnexus-cli/SKILL.md | 2 +- gitnexus/README.md | 23 + gitnexus/skills/gitnexus-cli.md | 2 +- gitnexus/src/cli/analyze-watch.ts | 504 +++++ gitnexus/src/cli/analyze.ts | 2 +- gitnexus/src/cli/auto-sync.ts | 125 ++ gitnexus/src/cli/help-i18n.ts | 2 + gitnexus/src/cli/i18n/en.ts | 10 + gitnexus/src/cli/i18n/zh-CN.ts | 10 + gitnexus/src/cli/index.ts | 16 + gitnexus/src/cli/watch.ts | 506 +---- .../core/auto-sync/analysis-worker-launch.ts | 210 ++ gitnexus/src/core/auto-sync/config.ts | 367 ++++ gitnexus/src/core/auto-sync/index.ts | 57 + gitnexus/src/core/auto-sync/path-security.ts | 286 +++ gitnexus/src/core/auto-sync/repo.ts | 7 + gitnexus/src/core/auto-sync/runner.ts | 561 +++++ gitnexus/src/core/auto-sync/starter.ts | 643 ++++++ gitnexus/src/core/auto-sync/state.ts | 173 ++ gitnexus/src/core/group/group-lock.ts | 11 +- gitnexus/src/core/lbug/lbug-adapter.ts | 15 +- .../src/server/analyze-worker-protocol.ts | 9 +- gitnexus/src/server/analyze-worker.ts | 60 +- gitnexus/src/server/api.ts | 4 +- gitnexus/src/server/git-clone.ts | 358 ++- gitnexus/src/storage/file-lock.ts | 189 ++ gitnexus/src/storage/repo-manager.ts | 31 +- gitnexus/src/utils/process-identity.ts | 40 + .../test/integration/watch-filesystem.test.ts | 2 +- .../unit/auto-sync-analysis-worker.test.ts | 232 ++ gitnexus/test/unit/auto-sync-runner.test.ts | 1949 +++++++++++++++++ gitnexus/test/unit/auto-sync.test.ts | 730 ++++++ gitnexus/test/unit/cli-index-help.test.ts | 111 +- gitnexus/test/unit/file-lock.test.ts | 227 ++ gitnexus/test/unit/git-clone.test.ts | 618 +++++- gitnexus/test/unit/hooks.test.ts | 30 +- gitnexus/test/unit/process-identity.test.ts | 42 + ...repo-manager-registry-atomic-write.test.ts | 4 +- gitnexus/test/unit/repo-manager.test.ts | 19 + gitnexus/test/unit/watch-command.test.ts | 43 + .../test/unit/watch-failure-policy.test.ts | 2 +- gitnexus/test/unit/watch-paths.test.ts | 2 +- 47 files changed, 7607 insertions(+), 682 deletions(-) create mode 100644 gitnexus/src/cli/analyze-watch.ts create mode 100644 gitnexus/src/cli/auto-sync.ts create mode 100644 gitnexus/src/core/auto-sync/analysis-worker-launch.ts create mode 100644 gitnexus/src/core/auto-sync/config.ts create mode 100644 gitnexus/src/core/auto-sync/index.ts create mode 100644 gitnexus/src/core/auto-sync/path-security.ts create mode 100644 gitnexus/src/core/auto-sync/repo.ts create mode 100644 gitnexus/src/core/auto-sync/runner.ts create mode 100644 gitnexus/src/core/auto-sync/starter.ts create mode 100644 gitnexus/src/core/auto-sync/state.ts create mode 100644 gitnexus/src/storage/file-lock.ts create mode 100644 gitnexus/src/utils/process-identity.ts create mode 100644 gitnexus/test/unit/auto-sync-analysis-worker.test.ts create mode 100644 gitnexus/test/unit/auto-sync-runner.test.ts create mode 100644 gitnexus/test/unit/auto-sync.test.ts create mode 100644 gitnexus/test/unit/file-lock.test.ts create mode 100644 gitnexus/test/unit/process-identity.test.ts create mode 100644 gitnexus/test/unit/watch-command.test.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index 170703439..be02d92fd 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -33,7 +33,7 @@ Run from the project root. This parses all source files, builds the knowledge gr For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. -Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. ### status — Check index freshness diff --git a/.gitignore b/.gitignore index e16544f71..3d125d475 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ npm-debug.log* # Testing coverage/ +.tmp-test/ +gitnexus/.tmp-test/ # Misc *.local diff --git a/Dockerfile.cli b/Dockerfile.cli index b42c22dad..1488bee54 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -51,8 +51,9 @@ RUN npm run postinstall --prefix gitnexus # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -# curl for the healthcheck; git for cloning; ca-certificates for TLS verification. -RUN apt-get update && apt-get install -y --no-install-recommends curl git ca-certificates && rm -rf /var/lib/apt/lists/* \ +# curl for the healthcheck; git for cloning; procps for watch process identity; +# ca-certificates for TLS verification. +RUN apt-get update && apt-get install -y --no-install-recommends curl git procps ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/local/lib/node_modules/npm \ && rm -rf /usr/local/lib/node_modules/corepack \ && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack diff --git a/README.md b/README.md index f011447fd..0e0e2f616 100644 --- a/README.md +++ b/README.md @@ -470,6 +470,46 @@ If embeddings are skipped on a large repository, the indexed graph likely exceed +
+Keep remote repositories indexed with gitnexus auto-sync + +`gitnexus auto-sync` clones or pulls configured repositories, analyzes new commits, and optionally syncs their group. It runs once immediately, then repeats on the configured interval. It runs in the foreground; use your process manager if it must survive a shell session. `gitnexus watch` is reserved and prints this split; it does not start auto-sync or local file watching. + +```bash +# 1. Create the config once. It never overwrites an existing file. +gitnexus auto-sync init + +# 2. Edit $GITNEXUS_HOME/watch_config.yml, then start it. +gitnexus auto-sync start # `gitnexus auto-sync` is equivalent +gitnexus auto-sync status +gitnexus auto-sync restart # Required after config changes +gitnexus auto-sync stop +gitnexus auto-sync reset # Clear failure state; leaves clones and indexes intact +``` + +`GITNEXUS_HOME` defaults to `~/.gitnexus`. A minimal configuration: + +```yaml +sync_interval_minutes: 10 +analyze_timeout: 5m +projects: + - local_path: /absolute/path/to/clones + branches: [main, master] + overwrite_local_changes: false + remote_urls: + - git@github.com:owner/repo.git +``` + +- `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. Clones are stored below it as `host/namespace/repo`. +- Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. +- `branches` are tried in order. The legacy `branch` field is supported, but do not set both. +- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. Timeout and `auto-sync stop` request safe cancellation; a worker in native work exits after reaching a JS-visible safe point. Until then, auto-sync reports `cancelling` or `stopping` and retains ownership so another auto-sync cannot take over, for up to 5 seconds — after that the parent stops waiting and leaves the worker to exit on its own rather than killing it mid-write. This behavior is the same on macOS and Windows. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten; setting it to `true` also deletes untracked files in the clone, while keeping ignored paths. +- Add `group_name` only after creating that group with `gitnexus group create `. Partial clone output is isolated and removed after 14 days. + +See the [full auto-sync configuration and runtime reference](gitnexus/README.md#gitnexus-auto-sync) for concurrency, timeouts, failure thresholds, and runtime files. + +
+
Repository groups (multi-repo / monorepo service tracking) diff --git a/eslint-rules/require-safe-parse.mjs b/eslint-rules/require-safe-parse.mjs index 4ab9dbd8a..4bad4280d 100644 --- a/eslint-rules/require-safe-parse.mjs +++ b/eslint-rules/require-safe-parse.mjs @@ -19,14 +19,14 @@ * * False-positive suppression: * - Skips calls whose receiver is a known non-tree-sitter library (`JSON`, - * `URL`, `marked`, `Number`). + * `URL`, `marked`, `Number`, `path`). * - Skips calls whose first argument is a string-literal (grammar-load smoke * tests like `_testParser.parse('service X { rpc Y (R) returns (R); }')`). * - Skips test files (`.test.ts`/`.test.tsx`/`.spec.ts`). * - Skips the `safe-parse.ts` helper itself. */ -const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math']); +const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math', 'path']); export default { meta: { @@ -74,7 +74,7 @@ export default { // Receiver-text-shape skip: anything matching well-known JS APIs that // happen to have a `.parse()` shape but aren't tree-sitter. if ( - /^(JSON|URL|marked|Number|Math|Date|globalThis\.JSON)\b/.test(receiverText) || + /^(JSON|URL|marked|Number|Math|Date|path|globalThis\.JSON)\b/.test(receiverText) || /\bjson\.parse\b/i.test(receiverText) ) { return; diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index 170703439..be02d92fd 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -33,7 +33,7 @@ Run from the project root. This parses all source files, builds the knowledge gr For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. -Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. ### status — Check index freshness diff --git a/gitnexus/README.md b/gitnexus/README.md index 12bd96d20..9abc122c5 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -249,6 +249,7 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) +gitnexus auto-sync [init|start|restart|stop|status|reset] # Scheduled remote clone/pull + analyze from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus index # Register an existing .gitnexus/ folder into the global registry @@ -309,6 +310,28 @@ and `serve` processes periodically check for a newly published index and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. +### `gitnexus auto-sync` + +`gitnexus auto-sync` is a different product from `gitnexus analyze --watch`. It is the explicit long-running auto-sync entrypoint that clones or pulls configured remotes. `gitnexus watch` is reserved and does not start either job: it prints this split. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus auto-sync init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus auto-sync` is the same as `gitnexus auto-sync start`; `restart`, `stop`, `status`, and `reset` manage the same `GITNEXUS_HOME` instance. `reset` removes only the derived analysis state and commit snapshot; clones, indexes, and registry entries are untouched. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.mutex` prevents multiple auto-sync processes for one home, `watch.owner.json` records ownership metadata, `watch.pid` plus `watch.status.json` expose process state, `watch.stop..json` is a temporary owner-fenced stop request, and `quarantine/` stores partial clone output before entries are removed after 14 days, keeping at most the five newest entries per repository regardless of age. Mutexes with verified dead owners are reclaimed automatically after an abnormal exit. Invalid or legacy mutexes fail closed; confirm no auto-sync process is running before manually removing `watch.mutex` and stale `watch.pid` / `watch.owner.json`. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_timeout: 5m +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + overwrite_local_changes: false + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal; each remote is cloned below it as `host/namespace/repo`, preventing same-basename repositories from colliding. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. It must not exceed one hour or `sync_interval_minutes`, whichever is smaller — so a bare `600000` is rejected, because it means 600000 seconds rather than milliseconds. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. Timeout and `auto-sync stop` request safe cancellation; a worker already in native work exits after it returns to a JS-visible safe point. While waiting, auto-sync reports `cancelling` or `stopping` and keeps its ownership files so another auto-sync cannot take over. The parent waits up to 5 seconds for the worker to exit; after that it stops waiting, releases its ownership files, and leaves the worker to finish and exit on its own rather than killing it mid-write. `auto-sync stop` uses this same control path on macOS and Windows. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes and additionally discards untracked files and directories in the clone after checkout — ignored paths, including GitNexus's own `.gitnexus/` storage, are preserved. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. Each analysis worker's heap cap is the machine-wide cap divided by the number of repositories analyzed in parallel, so concurrent workers share one memory budget instead of each claiming the whole machine. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit or `gitnexus auto-sync reset` clears the block and allows analysis again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. + GraphQL contract matching is opt-in in the group's `group.yaml`: ```yaml diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index 170703439..be02d92fd 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -33,7 +33,7 @@ Run from the project root. This parses all source files, builds the knowledge gr For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. -Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. ### status — Check index freshness diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts new file mode 100644 index 000000000..2e4744bc5 --- /dev/null +++ b/gitnexus/src/cli/analyze-watch.ts @@ -0,0 +1,504 @@ +/** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { watch, type FSWatcher } from 'chokidar'; +import { createWatchIgnorePredicate } from '../config/ignore-service.js'; +import { + analyzeFailureMayHaveMutatedLiveIndex, + runFullAnalysis, + type AnalyzeOptions as CoreAnalyzeOptions, + type AnalyzeResult, +} from '../core/run-analyze.js'; +import { getGitRoot, hasGitDir } from '../storage/git.js'; +import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; +import { GITNEXUS_DIR } from '../storage/repo-meta.js'; +import { + loadAnalyzeConfigStrict, + mergeAnalyzeOptions, + validateBranchName, +} from './analyze-config.js'; +import type { AnalyzeOptions } from './analyze-options.js'; +import { ensureHeap } from './analyze.js'; +import { cliError, cliInfo, cliWarn } from './cli-message.js'; +import { + WATCH_FULL_REFRESH_PATH, + WatchRefreshQueue, + type WatchRefreshError, +} from './watch-queue.js'; + +const DEFAULT_DEBOUNCE_MS = 300; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_FILE_SIZE_KB = 32 * 1024; +const TRANSIENT_WATCH_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'ENOTDIR', 'EPERM']); + +export type WatchCliOptions = AnalyzeOptions; + +function posixWatchPath(filePath: string): string { + return filePath.replace(/\\/g, '/').replace(/^\.\/+/, ''); +} + +export function isRelevantWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return ( + normalized.length > 0 && + normalized !== '.' && + !normalized.startsWith('../') && + !path.posix.isAbsolute(normalized) && + !path.win32.isAbsolute(filePath) + ); +} + +function isIgnoreControlPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return normalized === '.gitignore' || normalized === '.gitnexusignore'; +} + +function isConfigControlPath(filePath: string): boolean { + return posixWatchPath(filePath) === '.gitnexusrc'; +} + +function isAnalyzerOwnedWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath).replace(/\/+$/, ''); + return normalized === GITNEXUS_DIR || normalized.startsWith(`${GITNEXUS_DIR}/`); +} + +function repoRelativeWatchPath(repoPath: string, candidate: string): string | null { + const relative = path.relative(repoPath, candidate).replace(/\\/g, '/'); + if (!relative || relative.startsWith('../') || path.isAbsolute(relative)) return null; + return relative; +} + +export interface WatchEnvironmentBaseline { + readonly maxFileSize: string | undefined; + readonly workerTimeout: string | undefined; + readonly verbose: string | undefined; +} + +function setEnvironment(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +function positiveInteger( + value: string | undefined, + flag: string, + maximum?: number, +): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) + throw new Error(`${flag} must be a positive integer`); + if (maximum !== undefined && parsed > maximum) { + throw new Error(`${flag} must not exceed ${maximum}`); + } + return parsed; +} + +export async function resolveWatchOptions( + repoPath: string, + cli: WatchCliOptions, + baseline: WatchEnvironmentBaseline, + reportIgnoredConfig: (names: readonly string[]) => void = () => {}, +): Promise { + const config = (await loadAnalyzeConfigStrict(repoPath)) ?? {}; + const merged = mergeAnalyzeOptions(cli, config); + const unsupported = [ + ['--force', cli.force], + ['--repair-fts', cli.repairFts], + ['--embeddings', cli.embeddings], + ['--drop-embeddings', cli.dropEmbeddings], + ['--skills', cli.skills], + ['--default-branch', cli.defaultBranch], + ['--skip-agents-md', cli.skipAgentsMd], + ['--skip-skills', cli.skipSkills], + ['--no-stats', cli.stats === false], + ['--self-commit', cli.selfCommit], + ['--index-only', cli.indexOnly], + ['--skip-git', cli.skipGit], + ['--spring-actuator', cli.springActuator], + ['walCheckpointThreshold', cli.walCheckpointThreshold], + ['embeddingThreads', cli.embeddingThreads], + ['embeddingBatchSize', cli.embeddingBatchSize], + ['embeddingSubBatchSize', cli.embeddingSubBatchSize], + ['embeddingDevice', cli.embeddingDevice], + ['embeddingBaseUrl', cli.embeddingBaseUrl], + ['embeddingModel', cli.embeddingModel], + ['--embedding-auth-token', cli.embeddingAuthToken], + ['--embedding-dims', cli.embeddingDims], + ].filter(([, value]) => value !== undefined && value !== false); + if (unsupported.length > 0) { + throw new Error( + `analyze --watch does not support ${unsupported.map(([name]) => name).join(', ')}`, + ); + } + reportIgnoredConfig( + [ + ['embeddings', config.embeddings], + ['dropEmbeddings', config.dropEmbeddings], + ['defaultBranch', config.defaultBranch], + ['skipAgentsMd', config.skipAgentsMd !== undefined], + ['skipSkills', config.skipSkills !== undefined], + ['stats', config.stats !== undefined], + ['springActuator', config.springActuator], + ['walCheckpointThreshold', config.walCheckpointThreshold], + ['embeddingThreads', config.embeddingThreads], + ['embeddingBatchSize', config.embeddingBatchSize], + ['embeddingSubBatchSize', config.embeddingSubBatchSize], + ['embeddingDevice', config.embeddingDevice], + ['embeddingBaseUrl', config.embeddingBaseUrl], + ['embeddingModel', config.embeddingModel], + ] + .filter(([, value]) => value !== undefined && value !== false) + .map(([name]) => String(name)), + ); + const branch = + merged.branch === undefined ? undefined : validateBranchName(merged.branch, '--branch'); + const workerPoolSize = positiveInteger(merged.workers, '--workers'); + const workerTimeoutSeconds = positiveInteger(merged.workerTimeout, 'workerTimeout'); + const maxFileSize = positiveInteger(merged.maxFileSize, 'maxFileSize', MAX_FILE_SIZE_KB); + + setEnvironment( + 'GITNEXUS_MAX_FILE_SIZE', + maxFileSize === undefined ? baseline.maxFileSize : String(maxFileSize), + ); + if (workerTimeoutSeconds !== undefined) { + process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS = String(workerTimeoutSeconds * 1000); + } else { + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baseline.workerTimeout); + } + setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); + + return { + pdg: merged.pdg, + branch, + registryName: merged.name, + allowDuplicateName: merged.allowDuplicateName, + workerPoolSize, + fetchWrappers: merged.fetchWrappers, + skipAgentsMd: true, + skipSkills: true, + noStats: true, + atomicIncremental: process.platform !== 'win32', + }; +} + +function refreshSummary( + result: AnalyzeResult, + observedPaths: readonly string[], + durationMs: number, + lastSuccessfulRefreshAt: string, +): string { + const measured = result.incrementalStats; + const changed = measured?.changedFiles ?? (result.alreadyUpToDate ? 0 : observedPaths.length); + const reparsed = + measured?.reparsedFiles ?? + (typeof result.pipelineResult?.reparsedFileCount === 'number' + ? result.pipelineResult.reparsedFileCount + : 0); + const dependents = measured?.affectedDependents ?? 0; + const mode = measured?.writeMode ?? (result.alreadyUpToDate ? 'no-op' : 'full'); + return ( + `Refresh complete: ${changed} changed, ${reparsed} re-parsed, ` + + `${dependents} affected dependent(s), ${durationMs}ms, ${mode}; ` + + `last success ${lastSuccessfulRefreshAt}` + ); +} + +async function waitUntilReady(watcher: FSWatcher): Promise { + await new Promise((resolve, reject) => { + const ready = () => { + watcher.off('error', failed); + resolve(); + }; + const failed = (error: unknown) => { + watcher.off('ready', ready); + reject(error); + }; + watcher.once('ready', ready); + watcher.once('error', failed); + }); +} + +export interface WatchFileLoop { + readonly waitForIdle: () => Promise; + readonly close: () => Promise; +} + +class WatchControlReloadError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = 'WatchControlReloadError'; + } +} + +export function shouldStopAfterWatchRefreshFailure( + error: unknown, + paths: readonly string[], +): boolean { + return ( + paths.length > 0 && + !(error instanceof WatchControlReloadError) && + analyzeFailureMayHaveMutatedLiveIndex(error) + ); +} + +/** Start the real filesystem watcher with bounded, serialized refreshes. */ +export async function startWatchFileLoop( + repoPath: string, + debounceMs: number, + refresh: (paths: readonly string[]) => Promise, + onError: WatchRefreshError, + onWatcherError: (error: unknown) => void = (error) => onError(error, []), +): Promise { + let ignorePath = await createWatchIgnorePredicate(repoPath); + let ignoreControlValid = true; + const queue = new WatchRefreshQueue( + async (paths) => { + if (paths.some(isIgnoreControlPath) || !ignoreControlValid) { + const retryingInvalidControls = !ignoreControlValid; + try { + ignorePath = await createWatchIgnorePredicate(repoPath); + ignoreControlValid = true; + watcher.add(repoPath); + } catch (error) { + ignoreControlValid = false; + throw new WatchControlReloadError( + retryingInvalidControls + ? new Error( + 'Ignore controls remain invalid; fix them before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + await refresh(paths); + }, + onError, + debounceMs, + { + maxWaitMs: Math.max(2_000, debounceMs * 10), + maxPendingPaths: 1_000, + holdEventsUntilInitialRefresh: true, + isPriorityPath: (filePath) => isIgnoreControlPath(filePath) || isConfigControlPath(filePath), + }, + ); + + const watcher: FSWatcher = watch(repoPath, { + ignoreInitial: true, + atomic: true, + followSymlinks: false, + awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 20 }, + ignored: (candidate, stats) => { + const relative = repoRelativeWatchPath(repoPath, candidate); + if (relative !== null && isAnalyzerOwnedWatchPath(relative)) return true; + if (relative !== null && (isIgnoreControlPath(relative) || isConfigControlPath(relative))) { + return false; + } + return ignorePath(candidate, stats?.isDirectory() ?? false); + }, + }); + watcher.on('all', (event, changedPath) => { + if (event !== 'add' && event !== 'change' && event !== 'unlink') return; + const relative = repoRelativeWatchPath(repoPath, changedPath); + if (relative && isRelevantWatchPath(relative) && !isAnalyzerOwnedWatchPath(relative)) { + queue.enqueue(relative); + } + }); + watcher.on('error', (error) => { + // Chokidar can surface a transient EPERM on Windows while an ignored + // analyzer-owned path is replaced. Re-arm the root and force one bounded + // catch-up refresh so a missed event cannot leave the graph stale. Other + // watcher errors may mean coverage was lost and remain fatal. + if (TRANSIENT_WATCH_ERROR_CODES.has((error as NodeJS.ErrnoException).code ?? '')) { + watcher.add(repoPath); + queue.enqueue(WATCH_FULL_REFRESH_PATH); + return; + } + onWatcherError(error); + }); + + try { + await waitUntilReady(watcher); + await queue.runInitial(); + } catch (error) { + await watcher.close(); + await queue.close(); + throw error; + } + + return { + waitForIdle: () => queue.waitForIdle(), + close: async () => { + await watcher.close(); + await queue.close(); + }, + }; +} + +export async function watchCommandWithRunnerIdentity( + runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, + inputPath?: string, + cliOptions: WatchCliOptions = {}, +): Promise { + if (await ensureHeap({ cleanForwardedTermination: true })) return; + + const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { + cliError(' gitnexus analyze --watch requires a Git repository.'); + process.exitCode = 1; + return; + } + const repoPath = await fs.realpath(requestedRepoPath); + const baselineEnvironment: WatchEnvironmentBaseline = { + maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, + workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, + verbose: process.env.GITNEXUS_VERBOSE, + }; + try { + let ignoredConfigSignature: string | undefined; + const reportIgnoredConfig = (names: readonly string[]) => { + const signature = [...names].sort().join(','); + if (signature === ignoredConfigSignature) return; + ignoredConfigSignature = signature; + if (names.length > 0) { + cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); + } + }; + let debounceMs: number; + let analyzeOptions: CoreAnalyzeOptions; + try { + debounceMs = + positiveInteger( + cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), + '--debounce', + MAX_TIMER_DELAY_MS, + ) ?? DEFAULT_DEBOUNCE_MS; + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + } catch (error) { + cliError(` ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + return; + } + + let stopWatching!: () => void; + const stopped = new Promise((resolve) => { + stopWatching = resolve; + }); + const stop = () => stopWatching(); + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + let loop: WatchFileLoop; + let fatalRefreshError: unknown; + let configControlValid = true; + let lastSuccessfulRefreshAt: string | undefined; + try { + loop = await startWatchFileLoop( + repoPath, + debounceMs, + async (paths) => { + if (paths.some(isConfigControlPath) || !configControlValid) { + const retryingInvalidConfig = !configControlValid; + try { + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + configControlValid = true; + } catch (error) { + configControlValid = false; + throw new WatchControlReloadError( + retryingInvalidConfig + ? new Error( + 'Configuration remains invalid; fix it before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + const startedAt = Date.now(); + const result = await runFullAnalysis( + repoPath, + analyzeOptions, + { + onProgress: () => {}, + onLog: + process.env.GITNEXUS_VERBOSE === '1' + ? (message) => cliInfo(` ${message}`) + : undefined, + }, + runnerIdentityAtBootstrap, + ); + lastSuccessfulRefreshAt = new Date().toISOString(); + if (paths.length === 0) { + cliInfo( + result.alreadyUpToDate + ? `Watching ${repoPath}; index is up to date.` + : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, + ); + } else { + cliInfo( + refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), + ); + } + }, + (error, paths) => { + const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; + if (shouldStopAfterWatchRefreshFailure(error, paths)) { + fatalRefreshError = error; + cliError( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping because the live index may have been updated in place.', + ); + stopWatching(); + return; + } + const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; + cliWarn( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + `Retry scheduled; last success ${lastSuccess}.`, + ); + }, + (error) => { + fatalRefreshError = error; + cliError( + `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping.', + ); + stopWatching(); + }, + ); + } catch (error) { + cliError( + ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + + await stopped; + await loop.close(); + if (fatalRefreshError !== undefined) process.exitCode = 1; + } finally { + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + } + } finally { + setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); + setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); + } +} diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 1fa987320..beb553e01 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -776,7 +776,7 @@ export async function analyzeOrWatchCommandWithRunnerIdentity( options: AnalyzeOptions = {}, ): Promise { if (options.watch) { - const { watchCommandWithRunnerIdentity } = await import('./watch.js'); + const { watchCommandWithRunnerIdentity } = await import('./analyze-watch.js'); await watchCommandWithRunnerIdentity(runnerIdentityAtBootstrap, inputPath, options); return; } diff --git a/gitnexus/src/cli/auto-sync.ts b/gitnexus/src/cli/auto-sync.ts new file mode 100644 index 000000000..880deb5d5 --- /dev/null +++ b/gitnexus/src/cli/auto-sync.ts @@ -0,0 +1,125 @@ +/** Remote auto-sync CLI (`gitnexus auto-sync`). Local incremental watch lives in `analyze-watch.ts`. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + getAutoSyncConfigPath, + getAutoSyncMutexPath, + readAutoSyncWatchStatus, + resetAutoSyncState, + startAutoSyncWatch, + stopAutoSyncWatch, + type WatchStatusRecord, +} from '../core/auto-sync/index.js'; + +export async function autoSyncCommand(action = 'start'): Promise { + if (action === 'init') { + await initWatchConfig(); + return; + } + if (action === 'reset') { + if (!(await resetAutoSyncState())) { + process.stderr.write( + `[auto-sync] Cannot reset analysis state while the watch mutex is held. Confirm no watch process is running, then remove ${getAutoSyncMutexPath()}.\n`, + ); + process.exitCode = 1; + return; + } + process.stdout.write('[auto-sync] Reset analysis state.\n'); + return; + } + if (action === 'status') { + printStatus(await readAutoSyncWatchStatus()); + return; + } + if (action === 'stop') { + if ((await stopAutoSyncWatch()) !== 'stopped') process.exitCode = 1; + return; + } + if (action === 'restart') { + const result = await stopAutoSyncWatch(); + if (result === 'refused' || result === 'timeout') { + process.exitCode = 1; + return; + } + await startWatchProcess(); + return; + } + if (action !== 'start') { + process.stderr.write(`[auto-sync] Unknown auto-sync action: ${action}\n`); + process.exitCode = 1; + return; + } + await startWatchProcess(); +} + +async function startWatchProcess(): Promise { + const handle = await startAutoSyncWatch(); + if (!handle) { + process.exitCode = 1; + return; + } + + const stop = () => { + void handle.stop().then( + () => { + process.stderr.write('[auto-sync] Watch stopped.\n'); + process.exit(0); + }, + (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[auto-sync] Failed to stop watch: ${message}\n`); + process.exit(1); + }, + ); + }; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); +} + +function printStatus(status: WatchStatusRecord): void { + const parts = [`state=${status.state}`]; + if (status.pid) parts.push(`pid=${status.pid}`); + if (status.configPath) parts.push(`config=${status.configPath}`); + if (status.message) parts.push(`message=${status.message}`); + parts.push(`updated_at=${status.updatedAt}`); + process.stdout.write(`${parts.join(' ')}\n`); +} + +async function initWatchConfig(): Promise { + const configPath = getAutoSyncConfigPath(); + try { + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile( + configPath, + defaultSyncConfig(path.resolve(path.dirname(configPath), 'repos')), + { + flag: 'wx', + }, + ); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'EEXIST') { + process.stderr.write(`[auto-sync] Config already exists: ${configPath}\n`); + process.exitCode = 1; + return; + } + throw err; + } + process.stdout.write(`[auto-sync] Created ${configPath}\n`); +} + +function defaultSyncConfig(localPath: string): string { + return [ + 'sync_interval_minutes: 10', + 'max_concurrency: 1', + 'repo_git_timeout: 10s', + 'analyze_timeout: 5m', + 'analyze_failure_threshold: 3', + 'projects:', + ` - local_path: ${localPath}`, + ' branches: [master, main]', + ' overwrite_local_changes: false', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + '', + ].join('\n'); +} diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index cd4724d70..283e99832 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -13,6 +13,8 @@ const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', uninstall: 'help.command.uninstall.description', + watch: 'help.command.watch.description', + 'auto-sync': 'help.command.autoSync.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', serve: 'help.command.serve.description', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index c54f5707b..58904c48f 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -145,6 +145,16 @@ export const en = { 'One-time setup: configure MCP for Cursor, Claude Code, Antigravity, OpenCode, CodeBuddy, Qoder, Codex', 'help.command.uninstall.description': 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + 'help.command.autoSync.description': + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + 'help.autoSync.details': + '\nActions: init, start (default), restart, stop, status, reset\nConfiguration: GITNEXUS_HOME/watch_config.yml\nRuntime files: GITNEXUS_HOME/watch/watch.pid, watch.mutex, watch.owner.json, watch.status.json, auto-sync-state.json\nRecovery: mutexes with verified dead owners are reclaimed automatically; invalid or legacy mutexes fail closed and require manual removal after confirming no watch process is running.\nWrites: GITNEXUS_HOME/watch/project_commit_info.txt\nRemote URLs: only SSH URLs on github.com, gitlab.com, and gitee.com are allowed.\nRuns once immediately, then repeats on sync_interval_minutes.', + 'help.command.watch.description': + 'Ambiguous: use `analyze --watch` for local files, or `auto-sync` for scheduled remotes', + 'help.watch.details': + '\n`gitnexus watch` does not start a watcher.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', + 'error.watch.ambiguous': + '`gitnexus watch` is ambiguous.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index a8001af8a..de9249cc4 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -144,6 +144,16 @@ export const zhCN = { '一次性设置：为 Cursor、Claude Code、Antigravity、OpenCode、CodeBuddy、Qoder、Codex 配置 MCP', 'help.command.uninstall.description': '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', + 'help.command.autoSync.description': + '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', + 'help.autoSync.details': + '\n操作：init、start（默认）、restart、stop、status、reset\n配置：GITNEXUS_HOME/watch_config.yml\n运行时文件：GITNEXUS_HOME/watch/watch.pid、watch.mutex、watch.owner.json、watch.status.json、auto-sync-state.json\n恢复：已验证 owner 退出的 mutex 会自动回收；无效或旧版 mutex 会安全拒绝，确认没有 watch 进程运行后再手动删除。\n写入：GITNEXUS_HOME/watch/project_commit_info.txt\n远程地址：仅允许 github.com、gitlab.com 和 gitee.com 上的 SSH 地址。\n启动后立即运行一次，之后按 sync_interval_minutes 重复。', + 'help.command.watch.description': + '含义不明确：本地文件请用 `analyze --watch`，定时远程同步请用 `auto-sync`', + 'help.watch.details': + '\n`gitnexus watch` 不会启动监视器。\n 本地工作区增量索引：gitnexus analyze --watch\n 定时远程 clone/pull 并分析：gitnexus auto-sync start\n', + 'error.watch.ambiguous': + '`gitnexus watch` 含义不明确。\n 本地工作区增量索引：gitnexus analyze --watch\n 定时远程 clone/pull 并分析：gitnexus auto-sync start\n', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index e53312d62..d48bc65c9 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -45,6 +45,22 @@ program .option('-f, --force', 'Apply the changes (default is a dry-run preview)') .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); +program + .command('auto-sync [action]') + .description( + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + ) + .addHelpText('after', () => t('help.autoSync.details')) + .action(createLazyAction(() => import('./auto-sync.js'), 'autoSyncCommand')); + +program + .command('watch [action]') + .description( + 'Ambiguous: use `analyze --watch` for local files, or `auto-sync` for scheduled remotes', + ) + .addHelpText('after', () => t('help.watch.details')) + .action(createLazyAction(() => import('./watch.js'), 'watchAmbiguousCommand')); + // Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook // before it overwrites the var, so the postAction hook can restore it. The // analyzeCommand env snapshot is taken AFTER this hook runs, so it cannot undo diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts index 13212a56c..2a779b51a 100644 --- a/gitnexus/src/cli/watch.ts +++ b/gitnexus/src/cli/watch.ts @@ -1,503 +1,7 @@ -import path from 'node:path'; -import fs from 'node:fs/promises'; -import { watch, type FSWatcher } from 'chokidar'; -import { createWatchIgnorePredicate } from '../config/ignore-service.js'; -import { - analyzeFailureMayHaveMutatedLiveIndex, - runFullAnalysis, - type AnalyzeOptions as CoreAnalyzeOptions, - type AnalyzeResult, -} from '../core/run-analyze.js'; -import { getGitRoot, hasGitDir } from '../storage/git.js'; -import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; -import { GITNEXUS_DIR } from '../storage/repo-meta.js'; -import { - loadAnalyzeConfigStrict, - mergeAnalyzeOptions, - validateBranchName, -} from './analyze-config.js'; -import type { AnalyzeOptions } from './analyze-options.js'; -import { ensureHeap } from './analyze.js'; -import { cliError, cliInfo, cliWarn } from './cli-message.js'; -import { - WATCH_FULL_REFRESH_PATH, - WatchRefreshQueue, - type WatchRefreshError, -} from './watch-queue.js'; +/** Reserved CLI verb: never starts either watch product. */ +import { t } from './i18n/index.js'; -const DEFAULT_DEBOUNCE_MS = 300; -const MAX_TIMER_DELAY_MS = 2_147_483_647; -const MAX_FILE_SIZE_KB = 32 * 1024; -const TRANSIENT_WATCH_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'ENOTDIR', 'EPERM']); - -export type WatchCliOptions = AnalyzeOptions; - -function posixWatchPath(filePath: string): string { - return filePath.replace(/\\/g, '/').replace(/^\.\/+/, ''); -} - -export function isRelevantWatchPath(filePath: string): boolean { - const normalized = posixWatchPath(filePath); - return ( - normalized.length > 0 && - normalized !== '.' && - !normalized.startsWith('../') && - !path.posix.isAbsolute(normalized) && - !path.win32.isAbsolute(filePath) - ); -} - -function isIgnoreControlPath(filePath: string): boolean { - const normalized = posixWatchPath(filePath); - return normalized === '.gitignore' || normalized === '.gitnexusignore'; -} - -function isConfigControlPath(filePath: string): boolean { - return posixWatchPath(filePath) === '.gitnexusrc'; -} - -function isAnalyzerOwnedWatchPath(filePath: string): boolean { - const normalized = posixWatchPath(filePath).replace(/\/+$/, ''); - return normalized === GITNEXUS_DIR || normalized.startsWith(`${GITNEXUS_DIR}/`); -} - -function repoRelativeWatchPath(repoPath: string, candidate: string): string | null { - const relative = path.relative(repoPath, candidate).replace(/\\/g, '/'); - if (!relative || relative.startsWith('../') || path.isAbsolute(relative)) return null; - return relative; -} - -export interface WatchEnvironmentBaseline { - readonly maxFileSize: string | undefined; - readonly workerTimeout: string | undefined; - readonly verbose: string | undefined; -} - -function setEnvironment(name: string, value: string | undefined): void { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; -} - -function positiveInteger( - value: string | undefined, - flag: string, - maximum?: number, -): number | undefined { - if (value === undefined) return undefined; - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 1) - throw new Error(`${flag} must be a positive integer`); - if (maximum !== undefined && parsed > maximum) { - throw new Error(`${flag} must not exceed ${maximum}`); - } - return parsed; -} - -export async function resolveWatchOptions( - repoPath: string, - cli: WatchCliOptions, - baseline: WatchEnvironmentBaseline, - reportIgnoredConfig: (names: readonly string[]) => void = () => {}, -): Promise { - const config = (await loadAnalyzeConfigStrict(repoPath)) ?? {}; - const merged = mergeAnalyzeOptions(cli, config); - const unsupported = [ - ['--force', cli.force], - ['--repair-fts', cli.repairFts], - ['--embeddings', cli.embeddings], - ['--drop-embeddings', cli.dropEmbeddings], - ['--skills', cli.skills], - ['--default-branch', cli.defaultBranch], - ['--skip-agents-md', cli.skipAgentsMd], - ['--skip-skills', cli.skipSkills], - ['--no-stats', cli.stats === false], - ['--self-commit', cli.selfCommit], - ['--index-only', cli.indexOnly], - ['--skip-git', cli.skipGit], - ['--spring-actuator', cli.springActuator], - ['walCheckpointThreshold', cli.walCheckpointThreshold], - ['embeddingThreads', cli.embeddingThreads], - ['embeddingBatchSize', cli.embeddingBatchSize], - ['embeddingSubBatchSize', cli.embeddingSubBatchSize], - ['embeddingDevice', cli.embeddingDevice], - ['embeddingBaseUrl', cli.embeddingBaseUrl], - ['embeddingModel', cli.embeddingModel], - ['--embedding-auth-token', cli.embeddingAuthToken], - ['--embedding-dims', cli.embeddingDims], - ].filter(([, value]) => value !== undefined && value !== false); - if (unsupported.length > 0) { - throw new Error( - `analyze --watch does not support ${unsupported.map(([name]) => name).join(', ')}`, - ); - } - reportIgnoredConfig( - [ - ['embeddings', config.embeddings], - ['dropEmbeddings', config.dropEmbeddings], - ['defaultBranch', config.defaultBranch], - ['skipAgentsMd', config.skipAgentsMd !== undefined], - ['skipSkills', config.skipSkills !== undefined], - ['stats', config.stats !== undefined], - ['springActuator', config.springActuator], - ['walCheckpointThreshold', config.walCheckpointThreshold], - ['embeddingThreads', config.embeddingThreads], - ['embeddingBatchSize', config.embeddingBatchSize], - ['embeddingSubBatchSize', config.embeddingSubBatchSize], - ['embeddingDevice', config.embeddingDevice], - ['embeddingBaseUrl', config.embeddingBaseUrl], - ['embeddingModel', config.embeddingModel], - ] - .filter(([, value]) => value !== undefined && value !== false) - .map(([name]) => String(name)), - ); - const branch = - merged.branch === undefined ? undefined : validateBranchName(merged.branch, '--branch'); - const workerPoolSize = positiveInteger(merged.workers, '--workers'); - const workerTimeoutSeconds = positiveInteger(merged.workerTimeout, 'workerTimeout'); - const maxFileSize = positiveInteger(merged.maxFileSize, 'maxFileSize', MAX_FILE_SIZE_KB); - - setEnvironment( - 'GITNEXUS_MAX_FILE_SIZE', - maxFileSize === undefined ? baseline.maxFileSize : String(maxFileSize), - ); - if (workerTimeoutSeconds !== undefined) { - process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS = String(workerTimeoutSeconds * 1000); - } else { - setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baseline.workerTimeout); - } - setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); - - return { - pdg: merged.pdg, - branch, - registryName: merged.name, - allowDuplicateName: merged.allowDuplicateName, - workerPoolSize, - fetchWrappers: merged.fetchWrappers, - skipAgentsMd: true, - skipSkills: true, - noStats: true, - atomicIncremental: process.platform !== 'win32', - }; -} - -function refreshSummary( - result: AnalyzeResult, - observedPaths: readonly string[], - durationMs: number, - lastSuccessfulRefreshAt: string, -): string { - const measured = result.incrementalStats; - const changed = measured?.changedFiles ?? (result.alreadyUpToDate ? 0 : observedPaths.length); - const reparsed = - measured?.reparsedFiles ?? - (typeof result.pipelineResult?.reparsedFileCount === 'number' - ? result.pipelineResult.reparsedFileCount - : 0); - const dependents = measured?.affectedDependents ?? 0; - const mode = measured?.writeMode ?? (result.alreadyUpToDate ? 'no-op' : 'full'); - return ( - `Refresh complete: ${changed} changed, ${reparsed} re-parsed, ` + - `${dependents} affected dependent(s), ${durationMs}ms, ${mode}; ` + - `last success ${lastSuccessfulRefreshAt}` - ); -} - -async function waitUntilReady(watcher: FSWatcher): Promise { - await new Promise((resolve, reject) => { - const ready = () => { - watcher.off('error', failed); - resolve(); - }; - const failed = (error: unknown) => { - watcher.off('ready', ready); - reject(error); - }; - watcher.once('ready', ready); - watcher.once('error', failed); - }); -} - -export interface WatchFileLoop { - readonly waitForIdle: () => Promise; - readonly close: () => Promise; -} - -class WatchControlReloadError extends Error { - constructor(cause: unknown) { - super(cause instanceof Error ? cause.message : String(cause), { cause }); - this.name = 'WatchControlReloadError'; - } -} - -export function shouldStopAfterWatchRefreshFailure( - error: unknown, - paths: readonly string[], -): boolean { - return ( - paths.length > 0 && - !(error instanceof WatchControlReloadError) && - analyzeFailureMayHaveMutatedLiveIndex(error) - ); -} - -/** Start the real filesystem watcher with bounded, serialized refreshes. */ -export async function startWatchFileLoop( - repoPath: string, - debounceMs: number, - refresh: (paths: readonly string[]) => Promise, - onError: WatchRefreshError, - onWatcherError: (error: unknown) => void = (error) => onError(error, []), -): Promise { - let ignorePath = await createWatchIgnorePredicate(repoPath); - let ignoreControlValid = true; - const queue = new WatchRefreshQueue( - async (paths) => { - if (paths.some(isIgnoreControlPath) || !ignoreControlValid) { - const retryingInvalidControls = !ignoreControlValid; - try { - ignorePath = await createWatchIgnorePredicate(repoPath); - ignoreControlValid = true; - watcher.add(repoPath); - } catch (error) { - ignoreControlValid = false; - throw new WatchControlReloadError( - retryingInvalidControls - ? new Error( - 'Ignore controls remain invalid; fix them before indexing more changes.', - { - cause: error, - }, - ) - : error, - ); - } - } - await refresh(paths); - }, - onError, - debounceMs, - { - maxWaitMs: Math.max(2_000, debounceMs * 10), - maxPendingPaths: 1_000, - holdEventsUntilInitialRefresh: true, - isPriorityPath: (filePath) => isIgnoreControlPath(filePath) || isConfigControlPath(filePath), - }, - ); - - const watcher: FSWatcher = watch(repoPath, { - ignoreInitial: true, - atomic: true, - followSymlinks: false, - awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 20 }, - ignored: (candidate, stats) => { - const relative = repoRelativeWatchPath(repoPath, candidate); - if (relative !== null && isAnalyzerOwnedWatchPath(relative)) return true; - if (relative !== null && (isIgnoreControlPath(relative) || isConfigControlPath(relative))) { - return false; - } - return ignorePath(candidate, stats?.isDirectory() ?? false); - }, - }); - watcher.on('all', (event, changedPath) => { - if (event !== 'add' && event !== 'change' && event !== 'unlink') return; - const relative = repoRelativeWatchPath(repoPath, changedPath); - if (relative && isRelevantWatchPath(relative) && !isAnalyzerOwnedWatchPath(relative)) { - queue.enqueue(relative); - } - }); - watcher.on('error', (error) => { - // Chokidar can surface a transient EPERM on Windows while an ignored - // analyzer-owned path is replaced. Re-arm the root and force one bounded - // catch-up refresh so a missed event cannot leave the graph stale. Other - // watcher errors may mean coverage was lost and remain fatal. - if (TRANSIENT_WATCH_ERROR_CODES.has((error as NodeJS.ErrnoException).code ?? '')) { - watcher.add(repoPath); - queue.enqueue(WATCH_FULL_REFRESH_PATH); - return; - } - onWatcherError(error); - }); - - try { - await waitUntilReady(watcher); - await queue.runInitial(); - } catch (error) { - await watcher.close(); - await queue.close(); - throw error; - } - - return { - waitForIdle: () => queue.waitForIdle(), - close: async () => { - await watcher.close(); - await queue.close(); - }, - }; -} - -export async function watchCommandWithRunnerIdentity( - runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, - inputPath?: string, - cliOptions: WatchCliOptions = {}, -): Promise { - if (await ensureHeap({ cleanForwardedTermination: true })) return; - - const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); - if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { - cliError(' gitnexus analyze --watch requires a Git repository.'); - process.exitCode = 1; - return; - } - const repoPath = await fs.realpath(requestedRepoPath); - const baselineEnvironment: WatchEnvironmentBaseline = { - maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, - workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, - verbose: process.env.GITNEXUS_VERBOSE, - }; - try { - let ignoredConfigSignature: string | undefined; - const reportIgnoredConfig = (names: readonly string[]) => { - const signature = [...names].sort().join(','); - if (signature === ignoredConfigSignature) return; - ignoredConfigSignature = signature; - if (names.length > 0) { - cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); - } - }; - let debounceMs: number; - let analyzeOptions: CoreAnalyzeOptions; - try { - debounceMs = - positiveInteger( - cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), - '--debounce', - MAX_TIMER_DELAY_MS, - ) ?? DEFAULT_DEBOUNCE_MS; - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); - } catch (error) { - cliError(` ${error instanceof Error ? error.message : String(error)}`); - process.exitCode = 1; - return; - } - - let stopWatching!: () => void; - const stopped = new Promise((resolve) => { - stopWatching = resolve; - }); - const stop = () => stopWatching(); - process.once('SIGINT', stop); - process.once('SIGTERM', stop); - try { - let loop: WatchFileLoop; - let fatalRefreshError: unknown; - let configControlValid = true; - let lastSuccessfulRefreshAt: string | undefined; - try { - loop = await startWatchFileLoop( - repoPath, - debounceMs, - async (paths) => { - if (paths.some(isConfigControlPath) || !configControlValid) { - const retryingInvalidConfig = !configControlValid; - try { - analyzeOptions = await resolveWatchOptions( - repoPath, - cliOptions, - baselineEnvironment, - reportIgnoredConfig, - ); - configControlValid = true; - } catch (error) { - configControlValid = false; - throw new WatchControlReloadError( - retryingInvalidConfig - ? new Error( - 'Configuration remains invalid; fix it before indexing more changes.', - { - cause: error, - }, - ) - : error, - ); - } - } - const startedAt = Date.now(); - const result = await runFullAnalysis( - repoPath, - analyzeOptions, - { - onProgress: () => {}, - onLog: - process.env.GITNEXUS_VERBOSE === '1' - ? (message) => cliInfo(` ${message}`) - : undefined, - }, - runnerIdentityAtBootstrap, - ); - lastSuccessfulRefreshAt = new Date().toISOString(); - if (paths.length === 0) { - cliInfo( - result.alreadyUpToDate - ? `Watching ${repoPath}; index is up to date.` - : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, - ); - } else { - cliInfo( - refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), - ); - } - }, - (error, paths) => { - const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; - if (shouldStopAfterWatchRefreshFailure(error, paths)) { - fatalRefreshError = error; - cliError( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - 'Watch mode is stopping because the live index may have been updated in place.', - ); - stopWatching(); - return; - } - const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; - cliWarn( - `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - `Retry scheduled; last success ${lastSuccess}.`, - ); - }, - (error) => { - fatalRefreshError = error; - cliError( - `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + - 'Watch mode is stopping.', - ); - stopWatching(); - }, - ); - } catch (error) { - cliError( - ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, - ); - process.exitCode = 1; - return; - } - - await stopped; - await loop.close(); - if (fatalRefreshError !== undefined) process.exitCode = 1; - } finally { - process.removeListener('SIGINT', stop); - process.removeListener('SIGTERM', stop); - } - } finally { - setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); - setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); - setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); - } +export async function watchAmbiguousCommand(_action?: string): Promise { + process.stderr.write(t('error.watch.ambiguous')); + process.exitCode = 1; } diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts new file mode 100644 index 000000000..d57a251a6 --- /dev/null +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -0,0 +1,210 @@ +import { fork, type ChildProcess } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; +import type { WorkerMessage } from '../../server/analyze-worker-protocol.js'; +import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; + +const _require = createRequire(import.meta.url); +export type AutoSyncAnalysisRunner = ( + repoPath: string, + options: AnalyzeOptions, + timeoutMs: number, + signal?: AbortSignal, + onCancellationRequested?: () => void, + concurrency?: number, +) => Promise>; + +interface AnalysisWorker extends Pick { + stdout?: Pick | null; + stderr?: Pick | null; + unref?: () => void; + channel?: { unref(): void } | null; +} + +/** + * How long the parent keeps waiting after asking a worker to cancel. + * + * Must stay below `stopAutoSyncWatch`'s process-exit budget, or a worker wedged + * past its safe point still turns `watch stop` into a timeout. + */ +const AUTO_SYNC_CANCEL_GRACE_MS = 5_000; + +export interface AutoSyncAnalysisLaunchDeps { + forkWorker: (workerPath: string, execArgv: string[]) => AnalysisWorker; + setTimeoutFn: typeof setTimeout; + clearTimeoutFn: typeof clearTimeout; + cancelGraceMs: number; +} + +const DEFAULT_DEPS: AutoSyncAnalysisLaunchDeps = { + forkWorker: (workerPath, execArgv) => + fork(workerPath, [], { + execArgv, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }), + setTimeoutFn: setTimeout, + clearTimeoutFn: clearTimeout, + cancelGraceMs: AUTO_SYNC_CANCEL_GRACE_MS, +}; + +/** + * Per-worker V8 heap cap for one tick. + * + * `autoHeapCapMb()` is a whole-machine figure, so handing it to every fork + * over-commits memory by the parallelism factor. Admission already bounds + * parallelism to `floor(availableMemoryGB / 2)`, so dividing here keeps the sum + * of worker heaps inside the machine budget while leaving `max_concurrency` + * free to mean what it says. The two rules compose to a ~1.5GB per-worker floor. + */ +export function resolveWorkerHeapMb(concurrency = 1): number { + const slots = Number.isFinite(concurrency) && concurrency >= 1 ? Math.floor(concurrency) : 1; + return Math.max(1, Math.min(8192, Math.floor(autoHeapCapMb() / slots))); +} + +export function createAutoSyncAnalysisRunner( + overrides: Partial = {}, +): AutoSyncAnalysisRunner { + const deps = { ...DEFAULT_DEPS, ...overrides }; + return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency) => + new Promise>((resolve, reject) => { + if (signal?.aborted) { + reject(new Error('Analysis cancelled.')); + return; + } + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const workerPath = path.join( + path.dirname(callerPath), + '../../server', + isDev ? 'analyze-worker.ts' : 'analyze-worker.js', + ); + if (!existsSync(workerPath)) { + reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); + return; + } + const workerHeapMb = resolveWorkerHeapMb(concurrency); + const execArgv = isDev + ? [ + '--import', + pathToFileURL(_require.resolve('tsx/esm')).href, + `--max-old-space-size=${workerHeapMb}`, + ] + : [`--max-old-space-size=${workerHeapMb}`]; + const child = deps.forkWorker(workerPath, execArgv); + child.stdout?.resume(); + child.stderr?.resume(); + + let terminalOutcome: WorkerMessage | undefined; + let terminationError: Error | undefined; + let settled = false; + let graceTimer: ReturnType | undefined; + const cleanup = () => { + deps.clearTimeoutFn(timeout); + deps.clearTimeoutFn(graceTimer); + signal?.removeEventListener('abort', onAbort); + }; + // Stop the parent owning a worker it has given up waiting for. An + // established IPC channel keeps this event loop alive even after unref, + // so both handles have to go. Never a kill: the child may be inside + // native work and is left to reach its own safe point. + const releaseChild = () => { + child.channel?.unref?.(); + child.unref?.(); + }; + const settle = (error?: Error, result?: Pick) => { + if (settled) return; + settled = true; + cleanup(); + if (error) reject(error); + else resolve(result!); + }; + const requestCancellation = (error: Error) => { + if (settled || terminationError) return; + terminationError = error; + deps.clearTimeoutFn(timeout); + onCancellationRequested?.(); + // IPC has the same semantics on macOS and Windows. The worker exits only + // after reaching a JS-visible safe point; this parent keeps ownership until then. + try { + child.send({ type: 'cancel' }); + } catch { + // A closed IPC channel still has an exit/error path. Do not force-kill a + // worker that may be inside native code. + } + // Bounded wait. A worker stuck past its safe point would otherwise leave + // this promise pending forever, wedging `activeRun` so `stop()` — and the + // `watch stop` waiting on this process to exit — can never finish. Settle + // the parent's wait and drop the IPC channel's hold on this event loop; + // an established channel keeps the parent alive even after unref. The + // child is deliberately left running rather than killed mid-write. + graceTimer = deps.setTimeoutFn(() => { + if (settled) return; + releaseChild(); + settle( + new Error( + `${error.message} The analyze worker did not exit within ${deps.cancelGraceMs}ms; ` + + 'it was left running so its native work is not interrupted.', + ), + ); + }, deps.cancelGraceMs); + }; + const timeout = deps.setTimeoutFn( + () => requestCancellation(new Error(`Analysis timed out after ${timeoutMs}ms.`)), + timeoutMs, + ); + const onAbort = () => requestCancellation(new Error('Analysis cancelled.')); + signal?.addEventListener('abort', onAbort, { once: true }); + + child.on('message', (message: WorkerMessage) => { + // Once timeout/cancellation requested shutdown, its reason owns the + // result. A terminal IPC can already be queued behind cancellation. + if (message.type === 'progress' || terminalOutcome || terminationError) return; + terminalOutcome = message; + deps.clearTimeoutFn(timeout); + }); + child.on('error', (error) => { + const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); + requestCancellation(workerError); + // This settles immediately rather than waiting out the grace, so the + // grace timer that would otherwise have released the child is cleared + // by cleanup(). Release it here instead — an errored channel does not + // mean the worker stopped. + releaseChild(); + settle(workerError); + }); + child.on('exit', (code, childSignal) => { + if (settled) return; + if (terminationError) { + settle(terminationError); + return; + } + if (terminalOutcome?.type === 'complete') { + settle(undefined, { stats: terminalOutcome.result.stats }); + return; + } + if (terminalOutcome?.type === 'error') { + settle(new Error(terminalOutcome.message)); + return; + } + settle( + new Error( + `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, + ), + ); + }); + try { + child.send({ type: 'start', repoPath, options }); + } catch (error) { + const startError = new Error( + `Failed to start auto-sync analyze worker: ${(error as Error).message}`, + ); + requestCancellation(startError); + settle(startError); + } + }); +} + +export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner(); diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts new file mode 100644 index 000000000..fa91aa8b4 --- /dev/null +++ b/gitnexus/src/core/auto-sync/config.ts @@ -0,0 +1,367 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { normalizeConfiguredCloneRoot } from './path-security.js'; + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml'; +const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +const MIN_SYNC_INTERVAL_MINUTES = 5; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_SYNC_INTERVAL_MINUTES = Math.floor(MAX_TIMER_DELAY_MS / 60_000); +const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_CONCURRENCY = 1; +export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3; +const MIN_ANALYZE_FAILURE_THRESHOLD = 2; +const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']); + +/** + * A single clone/pull must fit inside one sync interval and inside an hour. + * This is also the guard for the unit slip the bare-number rule invites: + * `repo_git_timeout: 600000` means 600000 SECONDS (~7 days), which clears the + * Node timer ceiling and would silently disable the timeout. + */ +const MAX_REPO_GIT_TIMEOUT_MS = 3_600_000; + +// Mirrors REPO_NAME_PATTERN in server/git-clone.ts. Deliberately duplicated +// rather than imported: git-clone.ts already imports from this module, so the +// reverse edge would be a cycle. +const REMOTE_REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; +// Same charset for a namespace segment: GitLab subgroups allow exactly these, +// and excluding separators is what stops a segment smuggling in traversal. +const REMOTE_PATH_SEGMENT_PATTERN = REMOTE_REPO_NAME_PATTERN; + +export interface AutoSyncProjectConfig { + localPath: string; + groupName?: string; + overwriteLocalChanges: boolean; + branches: string[]; + remoteUrls: string[]; +} + +export interface AutoSyncConfig { + configPath: string; + syncIntervalMinutes: number; + repoGitTimeoutMs: number; + analyzeTimeoutMs: number; + maxConcurrency: number; + analyzeFailureThreshold: number; + projects: AutoSyncProjectConfig[]; +} + +export type AutoSyncConfigLoadResult = + | { ok: true; config: AutoSyncConfig } + | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; + +export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); +} + +export function parseBranchCandidates(branchValue: unknown): string[] { + const rawItems = Array.isArray(branchValue) + ? branchValue.flatMap((item) => String(item).split(',')) + : String(branchValue ?? '').split(','); + const branches: string[] = []; + const seen = new Set(); + for (const item of rawItems) { + const branch = item.trim(); + if (!branch || seen.has(branch)) continue; + seen.add(branch); + branches.push(branch); + } + return branches; +} + +export async function loadAutoSyncConfig( + configPath = getAutoSyncConfigPath(), +): Promise { + let content: string; + try { + content = await fs.readFile(configPath, 'utf-8'); + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${configPath}. Auto sync is skipped.`, + }; + } + return { + ok: false, + reason: 'unreadable', + message: `[auto-sync] Unable to read config file: ${configPath}. Auto sync is skipped.`, + }; + } + + try { + return { ok: true, config: parseAutoSyncConfig(content, configPath) }; + } catch (err: unknown) { + return { + ok: false, + reason: 'invalid', + message: `[auto-sync] Invalid watch_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + }; + } +} + +export function parseAutoSyncConfig(content: string, configPath: string): AutoSyncConfig { + const raw = yaml.load(content, { schema: yaml.JSON_SCHEMA }) as Record; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('expected a YAML object'); + } + + const errors: string[] = []; + const interval = Number(raw.sync_interval_minutes); + if (!Number.isInteger(interval) || interval <= 0) { + errors.push('sync_interval_minutes must be a positive integer'); + } else if (interval < MIN_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); + } else if (interval > MAX_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must not exceed ${MAX_SYNC_INTERVAL_MINUTES}`); + } + + // YAML booleans survive JSON_SCHEMA (`true`/`false`). `Number(true) === 1` + // would otherwise pass the integer check and silently mean concurrency 1. + let maxConcurrency = DEFAULT_MAX_CONCURRENCY; + if (raw.max_concurrency !== undefined) { + if (typeof raw.max_concurrency !== 'number' || !Number.isInteger(raw.max_concurrency)) { + errors.push('max_concurrency must be a positive integer'); + } else if (raw.max_concurrency <= 0) { + errors.push('max_concurrency must be a positive integer'); + } else { + maxConcurrency = raw.max_concurrency; + } + } + + const repoGitTimeoutMs = + raw.repo_git_timeout === undefined + ? DEFAULT_REPO_GIT_TIMEOUT_MS + : parseDurationMs(raw.repo_git_timeout); + const maxRepoGitTimeoutMs = + Number.isInteger(interval) && + interval >= MIN_SYNC_INTERVAL_MINUTES && + interval <= MAX_SYNC_INTERVAL_MINUTES + ? Math.min(interval * 60_000, MAX_REPO_GIT_TIMEOUT_MS) + : undefined; + if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) { + errors.push('repo_git_timeout must be a positive duration such as 10s'); + } else if (repoGitTimeoutMs > MAX_TIMER_DELAY_MS) { + errors.push(`repo_git_timeout must not exceed ${MAX_TIMER_DELAY_MS}ms`); + } else if (maxRepoGitTimeoutMs !== undefined && repoGitTimeoutMs > maxRepoGitTimeoutMs) { + errors.push( + `repo_git_timeout must not exceed ${maxRepoGitTimeoutMs}ms (the lesser of 1h and ` + + `sync_interval_minutes); a bare number is interpreted as seconds, so use an explicit ` + + `unit such as 600000ms or 10m`, + ); + } + + const maxAnalyzeTimeoutMs = + Number.isInteger(interval) && + interval >= MIN_SYNC_INTERVAL_MINUTES && + interval <= MAX_SYNC_INTERVAL_MINUTES + ? interval * 30_000 + : undefined; + const analyzeTimeoutMs = + raw.analyze_timeout === undefined + ? (maxAnalyzeTimeoutMs ?? 0) + : parseDurationMs(raw.analyze_timeout); + if (!Number.isInteger(analyzeTimeoutMs) || analyzeTimeoutMs <= 0) { + errors.push('analyze_timeout must be a positive duration such as 30m'); + } else if (maxAnalyzeTimeoutMs !== undefined && analyzeTimeoutMs > maxAnalyzeTimeoutMs) { + errors.push( + `analyze_timeout must not exceed half of sync_interval_minutes (${maxAnalyzeTimeoutMs / 60_000}m)`, + ); + } + + const analyzeFailureThreshold = + raw.analyze_failure_threshold === undefined + ? DEFAULT_ANALYZE_FAILURE_THRESHOLD + : Number(raw.analyze_failure_threshold); + if ( + !Number.isInteger(analyzeFailureThreshold) || + analyzeFailureThreshold < MIN_ANALYZE_FAILURE_THRESHOLD + ) { + errors.push(`analyze_failure_threshold must be an integer >= ${MIN_ANALYZE_FAILURE_THRESHOLD}`); + } + + const rawProjects = raw.projects; + if (!Array.isArray(rawProjects) || rawProjects.length === 0) { + errors.push('projects must contain at least one project'); + } + + const projects: AutoSyncProjectConfig[] = []; + if (Array.isArray(rawProjects)) { + rawProjects.forEach((projectValue, index) => { + const project = projectValue as Record; + if (!project || typeof project !== 'object' || Array.isArray(project)) { + errors.push(`projects[${index}] must be an object`); + return; + } + + const localPath = typeof project.local_path === 'string' ? project.local_path.trim() : ''; + if (!localPath) { + errors.push(`projects[${index}].local_path is required`); + } else { + try { + normalizeConfiguredCloneRoot(localPath); + } catch (err: unknown) { + errors.push(`projects[${index}].local_path ${(err as Error).message}`); + } + } + + const remoteUrls = Array.isArray(project.remote_urls) + ? project.remote_urls.map((url) => String(url).trim()).filter(Boolean) + : []; + if (remoteUrls.length === 0) { + errors.push(`projects[${index}].remote_urls must contain at least one URL`); + } + for (let urlIndex = 0; urlIndex < remoteUrls.length; urlIndex += 1) { + try { + validateAutoSyncRemoteUrl(remoteUrls[urlIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].remote_urls[${urlIndex}] ${(err as Error).message}`); + } + } + + if (project.branch !== undefined && project.branches !== undefined) { + errors.push(`projects[${index}] must not set both branch and branches`); + } + const branches = parseBranchCandidates( + project.branches !== undefined ? project.branches : project.branch, + ); + if (branches.length === 0) errors.push(`projects[${index}].branches is required`); + for (let branchIndex = 0; branchIndex < branches.length; branchIndex += 1) { + try { + validateAutoSyncBranchName(branches[branchIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].branches[${branchIndex}] ${(err as Error).message}`); + } + } + + const groupName = + typeof project.group_name === 'string' && project.group_name.trim() + ? project.group_name.trim() + : undefined; + if (groupName && !GROUP_NAME_PATTERN.test(groupName)) { + errors.push(`projects[${index}].group_name is invalid`); + } + + const overwriteLocalChanges = + project.overwrite_local_changes === undefined ? false : project.overwrite_local_changes; + if (typeof overwriteLocalChanges !== 'boolean') { + errors.push(`projects[${index}].overwrite_local_changes must be a boolean`); + } + + if (localPath && remoteUrls.length > 0 && branches.length > 0) { + projects.push({ + localPath, + groupName, + overwriteLocalChanges: overwriteLocalChanges === true, + branches, + remoteUrls, + }); + } + }); + } + + if (errors.length > 0) throw new Error(errors.join('; ')); + return { + configPath, + syncIntervalMinutes: interval, + repoGitTimeoutMs, + analyzeTimeoutMs, + maxConcurrency, + analyzeFailureThreshold, + projects, + }; +} + +export function validateAutoSyncRemoteUrl(remoteUrl: string): void { + const trimmed = remoteUrl.trim(); + if (trimmed.includes('?') || trimmed.includes('#')) { + throw new Error('must not include query strings or fragments'); + } + const match = /^git@([^:\s/]+):([^\s]+)$/.exec(trimmed); + if (!match) { + throw new Error('must use an SSH URL on github.com, gitlab.com, or gitee.com'); + } + const host = match[1].toLowerCase(); + const repoPath = match[2]; + if (!ALLOWED_REMOTE_HOSTS.has(host)) { + throw new Error('host must be one of github.com, gitlab.com, or gitee.com'); + } + const pathParts = repoPath.split('/'); + // Every segment becomes a directory component: the namespace segments build + // the clone path and the last one names the repo. So each is held to the same + // charset, which is what keeps a separator out of a segment — on Windows + // `..\..\outside` is traversal even though the segment is not literally `..`, + // and testing the raw string for `..` instead would reject an ordinary + // `foo..bar`. Traversal is a whole segment; a separator is a character. + const namespaceParts = pathParts.slice(0, -1); + if ( + repoPath.startsWith('/') || + pathParts.length < 2 || + pathParts.some((part) => !part || part === '.' || part === '..') || + namespaceParts.some((part) => !REMOTE_PATH_SEGMENT_PATTERN.test(part)) + ) { + throw new Error('path must include owner/repo without traversal'); + } + // The final segment becomes the on-disk clone directory via `extractRepoName`, + // whose name rules are stricter than the path check above: a backslash — or + // anything outside `[A-Za-z0-9._-]` — passes here and then throws once per + // tick inside the sync loop instead of at config load. These rules are a + // strict superset, so anything accepted here is accepted there. + const lastSegment = pathParts[pathParts.length - 1]; + const repoName = /\.git$/i.test(lastSegment) ? lastSegment.slice(0, -4) : lastSegment; + if ( + !repoName || + repoName === '.' || + repoName === '..' || + repoName === 'unknown' || + repoName.startsWith('-') || + !REMOTE_REPO_NAME_PATTERN.test(repoName) + ) { + throw new Error( + 'repository name must use only letters, digits, ".", "_", or "-" and must not be "unknown"', + ); + } +} + +export function validateAutoSyncBranchName(branch: string): void { + if (!branch.trim()) throw new Error('must not be empty'); + if (/[\s\0-\x1f\x7f]/.test(branch)) + throw new Error('must not contain whitespace or control characters'); + if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref'); + if (branch.startsWith('-')) throw new Error('must not start with "-"'); + if (branch.startsWith('/')) throw new Error('must not start with "/"'); + if (branch.includes('..')) throw new Error('must not contain ".."'); + if (branch.includes('`')) throw new Error('must not contain backticks'); + if (branch.endsWith('/') || branch.endsWith('.')) throw new Error('must not end with "/" or "."'); + if (branch.includes('//')) throw new Error('must not contain consecutive slashes'); + if (branch.includes('@{')) throw new Error('must not contain "@{"'); + if ( + branch + .split('/') + .some( + (component) => + component.startsWith('.') || component.endsWith('.') || component.endsWith('.lock'), + ) + ) + throw new Error('must not contain hidden, trailing-dot, or .lock path components'); +} + +export function parseDurationMs(value: unknown): number { + if (typeof value === 'number') return value * 1_000; + const raw = String(value ?? '').trim(); + const match = /^(\d+)(ms|s|m)?$/.exec(raw); + if (!match) return Number.NaN; + const amount = Number(match[1]); + const unit = match[2] ?? 's'; + if (unit === 'ms') return amount; + if (unit === 's') return amount * 1_000; + return amount * 60_000; +} diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts new file mode 100644 index 000000000..b02948779 --- /dev/null +++ b/gitnexus/src/core/auto-sync/index.ts @@ -0,0 +1,57 @@ +export { + AUTO_SYNC_CONFIG_FILE, + getAutoSyncConfigPath, + loadAutoSyncConfig, + parseAutoSyncConfig, + parseBranchCandidates, + parseDurationMs, + validateAutoSyncBranchName, + validateAutoSyncRemoteUrl, + type AutoSyncConfig, + type AutoSyncConfigLoadResult, + type AutoSyncProjectConfig, +} from './config.js'; +export { + buildStateKey, + getAutoSyncMutexPath, + getAutoSyncWatchDir, + getAutoSyncStatePath, + getProjectCommitInfoPath, + loadAutoSyncState, + resetAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + writeProjectCommitInfo, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitState, + type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, +} from './state.js'; +export { extractRepoNameFromRemoteUrl } from './repo.js'; +export { + normalizeConfiguredCloneRoot, + quarantineAutoSyncPartial, + resolveConfiguredCloneRoot, + type AutoSyncCloneRoot, +} from './path-security.js'; +export { + addRepoToGroup, + getAutoSyncRepoIdentity, + getConfiguredRepoPath, + resolveActualConcurrency, + runAutoSyncOnce, + syncGroupByName, + type AutoSyncLogger, + type AutoSyncRunDeps, + type AutoSyncRunResult, +} from './runner.js'; +export { + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + startAutoSyncWatch, + stopAutoSyncWatch, + type AutoSyncStartHandle, + type AutoSyncWatchStopResult, + type AutoSyncWatchPaths, + type WatchStatusRecord, +} from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts new file mode 100644 index 000000000..ed9d9201b --- /dev/null +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -0,0 +1,286 @@ +import fs from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { getAutoSyncWatchDir } from './state.js'; + +const WINDOWS_DANGEROUS_ROOTS = + process.platform === 'win32' + ? [ + process.env.SystemRoot, + process.env.ProgramData, + process.env.ProgramFiles, + process.env['ProgramFiles(x86)'], + ].filter((entry): entry is string => Boolean(entry)) + : []; + +const DANGEROUS_ROOTS = new Set( + [ + '/', + os.homedir(), + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ...WINDOWS_DANGEROUS_ROOTS, + ].map((entry) => path.resolve(entry)), +); + +const DANGEROUS_PARENT_ROOTS = new Set( + [ + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ...WINDOWS_DANGEROUS_ROOTS, + ].map((entry) => path.resolve(entry)), +); + +const QUARANTINE_RETENTION_DAYS = 14; +const QUARANTINE_MAX_ENTRIES_PER_REPO = 5; + +// `auto-sync----` — see quarantineAutoSyncPartial. +// The UUID is the only fixed-shape field, so it anchors the grouping key, and +// everything after it is the basename (`[A-Za-z0-9._-]` by construction). +const QUARANTINE_ENTRY_PATTERN = + /^auto-sync-.+-\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-(.+)$/i; + +export interface AutoSyncCloneRoot { + root: string; + quarantineRoot: string; + quarantineRetentionDays: number; +} + +export async function resolveConfiguredCloneRoot(localPath: string): Promise { + const root = normalizeConfiguredCloneRoot(localPath); + assertNotDangerousRoot(root); + await assertNoSymlinkPath(root); + await fs.mkdir(root, { recursive: true }); + await assertDirectoryOwnerAndPermissions(root); + const realRoot = await fs.realpath(root); + assertContainedOrSame( + root, + realRoot, + 'Configured clone root realpath escaped its normalized path', + ); + assertNotDangerousRoot(realRoot); + assertNotGitNexusInternalRoot(realRoot); + const quarantineRoot = path.join(getAutoSyncWatchDir(), 'quarantine'); + await pruneQuarantineEntries(quarantineRoot); + + return { + root: realRoot, + quarantineRoot, + quarantineRetentionDays: QUARANTINE_RETENTION_DAYS, + }; +} + +export function normalizeConfiguredCloneRoot(localPath: string): string { + const value = localPath.trim(); + if (!value) throw new Error('local_path is required'); + if (!path.isAbsolute(value)) throw new Error('local_path must be an absolute path'); + if (value.split(path.sep).includes('..')) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + const resolved = path.resolve(value); + if (resolved !== path.normalize(value)) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + return resolved; +} + +export async function quarantineAutoSyncPartial( + targetDir: string, + quarantineRoot: string, +): Promise { + await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); + const base = path.basename(targetDir); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const destination = path.join( + quarantineRoot, + `auto-sync-${stamp}-${process.pid}-${randomUUID()}-${base}`, + ); + try { + await fs.rename(targetDir, destination); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'EXDEV') throw err; + await fs.cp(targetDir, destination, { recursive: true }); + await fs.rm(targetDir, { recursive: true, force: true }); + } + await fs.writeFile( + `${destination}.README.txt`, + [ + 'GitNexus auto-sync isolated a partial or unsafe clone result.', + `Created at: ${new Date().toISOString()}`, + `Original path: ${targetDir}`, + `Retention: keep for ${QUARANTINE_RETENTION_DAYS} days unless an operator reviews and removes it earlier.`, + 'Cleanup: verify the original path and remote before manual deletion.', + '', + ].join('\n'), + 'utf-8', + ); + return destination; +} + +async function pruneQuarantineEntries(quarantineRoot: string): Promise { + const cutoff = Date.now() - QUARANTINE_RETENTION_DAYS * 24 * 60 * 60 * 1_000; + // readdir and stat both resolve through a link, so a symlinked quarantine + // root would age-sweep and delete entries somewhere else entirely. + const rootStat = await fs.lstat(quarantineRoot).catch(() => undefined); + if (rootStat?.isSymbolicLink()) { + throw new Error(`Refusing symlinked auto-sync quarantine root: ${quarantineRoot}`); + } + let entries; + try { + entries = await fs.readdir(quarantineRoot); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return; + throw err; + } + const survivors = ( + await Promise.all( + entries + .filter((entry) => entry.startsWith('auto-sync-')) + .map(async (entry) => { + const entryPath = path.join(quarantineRoot, entry); + const stat = await fs.stat(entryPath).catch(() => undefined); + if (stat && stat.mtimeMs < cutoff) { + await fs.rm(entryPath, { recursive: true, force: true }); + return undefined; + } + return entry; + }), + ) + ).filter((entry): entry is string => entry !== undefined); + + // Age alone never bounds a repo that fails on every tick: one partial clone + // per tick stays inside the retention window forever. Keep the newest few per + // repo. Entries that do not match the generated naming scheme (operator + // notes, names from another version) are left to the age sweep alone. + const byRepo = new Map(); + for (const entry of survivors) { + if (entry.endsWith('.README.txt')) continue; + const repo = QUARANTINE_ENTRY_PATTERN.exec(entry)?.[1]; + if (!repo) continue; + const group = byRepo.get(repo) ?? []; + group.push(entry); + byRepo.set(repo, group); + } + await Promise.all( + [...byRepo.values()].flatMap((group) => + group + // The timestamp is the leading fixed-width field, so a descending + // string sort is newest-first. + .sort((a, b) => (a < b ? 1 : a > b ? -1 : 0)) + .slice(QUARANTINE_MAX_ENTRIES_PER_REPO) + .map(async (entry) => { + await fs.rm(path.join(quarantineRoot, entry), { recursive: true, force: true }); + await fs.rm(path.join(quarantineRoot, `${entry}.README.txt`), { force: true }); + }), + ), + ); +} + +function assertNotDangerousRoot(root: string): void { + if (root === path.resolve(getGlobalDir(), 'repos')) return; + if (DANGEROUS_ROOTS.has(root)) throw new Error(`Refusing unsafe auto-sync clone root: ${root}`); + for (const dangerousRoot of DANGEROUS_PARENT_ROOTS) { + const rel = path.relative(dangerousRoot, root); + if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) { + throw new Error(`Refusing unsafe auto-sync clone root under ${dangerousRoot}: ${root}`); + } + } + if (path.parse(root).root === root) + throw new Error(`Refusing filesystem root as clone root: ${root}`); +} + +function assertNotGitNexusInternalRoot(root: string): void { + const gitnexusDir = path.resolve(getGlobalDir()); + const blocked = [ + path.join(gitnexusDir, 'groups'), + path.join(gitnexusDir, 'indexes'), + path.join(gitnexusDir, 'quarantine'), + path.join(getAutoSyncWatchDir(gitnexusDir), 'quarantine'), + ]; + for (const blockedRoot of blocked) { + const rel = path.relative(blockedRoot, root); + if (!rel || (!rel.startsWith('..') && !path.isAbsolute(rel))) { + throw new Error(`Refusing GitNexus internal directory as auto-sync clone root: ${root}`); + } + } +} + +async function assertNoSymlinkPath(root: string): Promise { + const parsed = path.parse(root); + let current = parsed.root; + const parts = root.slice(parsed.root.length).split(path.sep).filter(Boolean); + for (const part of parts) { + current = path.join(current, part); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) + throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); + } +} + +export async function assertDirectoryOwnerAndPermissions(root: string): Promise { + const stat = await fs.stat(root); + if (!stat.isDirectory()) throw new Error(`auto-sync clone root is not a directory: ${root}`); + // POSIX uid/mode have no meaning on Windows, and this runs on every tick for + // every project, so throwing here failed 100% of repos forever while `watch + // status` still read `running`. Skip the ownership assertions rather than the + // whole feature: the caller's other guards — dangerous-root rejection + // (including the Windows system roots), symlink refusal, realpath containment + // and the GitNexus-internal-root check — all still apply, and managed git runs + // with `core.hooksPath` pinned to the null device. + if (process.platform === 'win32') return; + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`auto-sync clone root is owned by uid ${stat.uid}, not current process uid`); + } + const mode = stat.mode & 0o777; + const groupWritable = (mode & 0o020) !== 0; + const worldWritable = (mode & 0o002) !== 0; + if (worldWritable) { + throw new Error(`Refusing world-writable auto-sync clone root: ${root}`); + } + if (groupWritable) { + throw new Error(`Refusing group-writable auto-sync clone root: ${root}`); + } +} + +function assertContainedOrSame(root: string, child: string, message: string): void { + const rel = path.relative(root, child); + if (rel.startsWith('..') || path.isAbsolute(rel)) throw new Error(message); +} diff --git a/gitnexus/src/core/auto-sync/repo.ts b/gitnexus/src/core/auto-sync/repo.ts new file mode 100644 index 000000000..ed8e1731d --- /dev/null +++ b/gitnexus/src/core/auto-sync/repo.ts @@ -0,0 +1,7 @@ +import { extractRepoName } from '../../server/git-clone.js'; +import { validateAutoSyncRemoteUrl } from './config.js'; + +export function extractRepoNameFromRemoteUrl(remoteUrl: string): string { + validateAutoSyncRemoteUrl(remoteUrl); + return extractRepoName(remoteUrl); +} diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts new file mode 100644 index 000000000..ed5f7afd7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -0,0 +1,561 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { loadGroupConfig } from '../group/config-parser.js'; +import { getDefaultGitnexusDir, getGroupDir } from '../group/storage.js'; +import { syncGroup } from '../group/sync.js'; +import { registerRepo, resolveBranchPlacement, type RepoMeta } from '../../storage/repo-manager.js'; +import { extractRepoNameFromRemoteUrl } from './repo.js'; +import { cloneOrPull, runGit } from '../../server/git-clone.js'; +import { resolveConfiguredCloneRoot } from './path-security.js'; +import { + buildStateKey, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + writeProjectCommitInfo, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, +} from './state.js'; +import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js'; +import { validateAutoSyncRemoteUrl } from './config.js'; +import { runAutoSyncAnalysis, type AutoSyncAnalysisRunner } from './analysis-worker-launch.js'; + +export interface AutoSyncLogger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export interface AutoSyncRunDeps { + cloneOrPull: typeof cloneOrPull; + getCurrentBranch: (repoPath: string, timeoutMs: number) => Promise; + getCurrentCommit: (repoPath: string, timeoutMs: number) => Promise; + runAnalysis: AutoSyncAnalysisRunner; + registerRepo: typeof registerRepo; + resolveBranchPlacement: typeof resolveBranchPlacement; + loadState: typeof loadAutoSyncState; + saveState: typeof saveAutoSyncState; + writeCommitInfo: typeof writeProjectCommitInfo; + addRepoToGroup: typeof addRepoToGroup; + syncGroupByName: typeof syncGroupByName; + resolveCloneRoot: typeof resolveConfiguredCloneRoot; + getAvailableMemoryGB: () => number; +} + +export interface AutoSyncRunResult { + synced: number; + analyzed: number; + skippedAnalysis: number; + failed: number; +} + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +const DEFAULT_LOGGER: AutoSyncLogger = { + info: (message) => process.stderr.write(`${message}\n`), + warn: (message) => process.stderr.write(`${message}\n`), + error: (message) => process.stderr.write(`${message}\n`), +}; + +const DEFAULT_DEPS: AutoSyncRunDeps = { + cloneOrPull, + getCurrentBranch: async (repoPath, timeoutMs) => { + const branch = (await runGit(['branch', '--show-current'], repoPath, { timeoutMs })).trim(); + return branch || undefined; + }, + getCurrentCommit: async (repoPath, timeoutMs) => + (await runGit(['rev-parse', 'HEAD'], repoPath, { timeoutMs })).trim(), + runAnalysis: runAutoSyncAnalysis, + registerRepo, + resolveBranchPlacement, + loadState: loadAutoSyncState, + saveState: saveAutoSyncState, + writeCommitInfo: writeProjectCommitInfo, + addRepoToGroup, + syncGroupByName, + resolveCloneRoot: resolveConfiguredCloneRoot, + getAvailableMemoryGB: () => Math.floor(process.availableMemory?.() ?? 0) / 1024 / 1024 / 1024, +}; + +export async function runAutoSyncOnce( + config: AutoSyncConfig, + options: { + deps?: Partial; + logger?: AutoSyncLogger; + now?: () => Date; + signal?: AbortSignal; + onAnalysisCancellationRequested?: () => void; + } = {}, +): Promise { + const deps = { ...DEFAULT_DEPS, ...options.deps }; + const logger = options.logger ?? DEFAULT_LOGGER; + const now = options.now ?? (() => new Date()); + throwIfAborted(options.signal); + const state = await deps.loadState(); + throwIfAborted(options.signal); + const groupsToSync = new Set(); + const groupStateKeys = new Map(); + const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + const commitInfoEntries: ProjectCommitInfoEntry[] = []; + const actualConcurrency = resolveActualConcurrency( + config.maxConcurrency, + deps.getAvailableMemoryGB(), + ); + logger.info( + `[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`, + ); + + const workItems = await buildWorkItems(config, deps); + // What will actually run at once. One repo means one worker, so the common + // single-project case still hands that worker the whole machine budget. + const analysisParallelism = Math.max(1, Math.min(actualConcurrency, workItems.length)); + const repoResults = await mapWithConcurrency( + workItems, + actualConcurrency, + options.signal, + async (item) => { + const lastSyncTime = now().toISOString(); + try { + throwIfAborted(options.signal); + if (!item.cloneRoot || !item.repoName || !item.targetDir) { + throw new Error(item.error ?? 'Invalid auto-sync work item'); + } + const repoName = item.repoName; + const targetDir = item.targetDir; + const syncResult = await syncFirstAvailableBranch({ + item, + repoName, + targetDir, + timeoutMs: config.repoGitTimeoutMs, + deps, + logger, + }); + throwIfAborted(options.signal); + if (syncResult.ok === false) { + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir, + branch: item.project.branches[0], + status: syncResult.status, + analyzeConsecutiveFailures: 0, + lastSyncTime, + }; + } + + const currentBranch = syncResult.branch; + + const currentCommit = await deps.getCurrentCommit(targetDir, config.repoGitTimeoutMs); + const stateKey = buildStateKey(targetDir, currentBranch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; + let lastAnalyzeError = previous?.lastAnalyzeError; + const groupSyncPending = previous?.groupSyncPending === true; + let stats: RepoMeta['stats'] | undefined; + + if (previous && previous.codeCommitId !== currentCommit) { + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } + + if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { + analyzeStatus = 'threshold_skipped'; + logger.error( + `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, + ); + } else if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + options.onAnalysisCancellationRequested, + analysisParallelism, + ); + throwIfAborted(options.signal); + stats = analysis.stats; + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; + analyzeStatus = 'failed'; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = shortErrorMessage(err); + logger.error( + `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, + ); + } + } else { + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + throwIfAborted(options.signal); + + return { + kind: 'synced' as const, + project: item.project, + repoName, + remoteUrl: item.remoteUrl, + targetDir, + branch: currentBranch, + currentCommit, + analyzedCommitId, + analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + groupSyncPending, + stats, + stateKey, + lastSyncTime, + }; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir: item.targetDir ?? '', + status: 'sync_failed' as const, + lastSyncTime, + }; + } + }, + ); + + for (const repoResult of repoResults) { + if (repoResult.kind === 'failed') { + result.failed += 1; + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + status: repoResult.status, + lastSyncTime: repoResult.lastSyncTime, + }); + continue; + } + + result.synced += 1; + let analyzeStatus = repoResult.analyzeStatus; + let analyzeConsecutiveFailures = repoResult.analyzeConsecutiveFailures; + let lastAnalyzeError = repoResult.lastAnalyzeError; + let analyzedCommitId = repoResult.analyzedCommitId; + if (analyzeStatus === 'success') { + const meta: RepoMeta = { + repoPath: repoResult.targetDir, + lastCommit: repoResult.currentCommit, + indexedAt: repoResult.lastSyncTime, + stats: repoResult.stats!, + branch: repoResult.branch, + remoteUrl: repoResult.remoteUrl, + }; + try { + // Reproduce the placement the analyze worker already made. Registering + // without a branch always takes the primary/flat arm, which relabels a + // pinned branch entry with whatever this tick happened to sync — visible + // on the documented branch-fallback path. + const placement = await deps.resolveBranchPlacement( + repoResult.targetDir, + repoResult.branch, + ); + await deps.registerRepo(repoResult.targetDir, meta, { + name: getAutoSyncRepoIdentity(repoResult.remoteUrl), + // Omitted rather than passed as undefined, so a primary index is + // registered with the same option shape it had before this branch. + ...(placement.branch ? { branch: placement.branch } : {}), + }); + result.analyzed += 1; + } catch (err: unknown) { + analyzeStatus = 'failed'; + analyzedCommitId = undefined; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = `Repository registration failed: ${shortErrorMessage(err)}`; + result.failed += 1; + logger.error(`[auto-sync] ${lastAnalyzeError}`); + } + } else if (analyzeStatus === 'failed') { + result.failed += 1; + } else { + result.skippedAnalysis += 1; + } + + const stateEntry: AutoSyncCommitStateEntry = { + codeCommitId: repoResult.currentCommit, + analyzedCommitId, + lastAnalyzeStatus: analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + groupSyncPending: repoResult.groupSyncPending, + lastSyncTime: repoResult.lastSyncTime, + }; + state[repoResult.stateKey] = stateEntry; + + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + codeCommitId: repoResult.currentCommit, + analyzedCommitId, + status: analyzeStatus, + analyzeConsecutiveFailures, + analyzeFailureThreshold: config.analyzeFailureThreshold, + lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }); + + if (repoResult.project.groupName) { + let groupMembershipOk = false; + let membershipAdded = false; + try { + membershipAdded = await deps.addRepoToGroup( + repoResult.project, + getAutoSyncRepoIdentity(repoResult.remoteUrl), + getAutoSyncRepoIdentity(repoResult.remoteUrl), + ); + groupMembershipOk = true; + } catch (err: unknown) { + result.failed += 1; + logger.error( + `[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`, + ); + } + if ( + groupMembershipOk && + (analyzeStatus === 'success' || + (membershipAdded && analyzeStatus === 'skipped') || + (analyzeStatus === 'skipped' && repoResult.groupSyncPending)) + ) { + const groupName = repoResult.project.groupName; + groupsToSync.add(groupName); + const keys = groupStateKeys.get(groupName) ?? []; + keys.push(repoResult.stateKey); + groupStateKeys.set(groupName, keys); + } + } + } + + await deps.saveState(state); + await deps.writeCommitInfo(commitInfoEntries); + let groupStateChanged = false; + for (const groupName of groupsToSync) { + try { + await deps.syncGroupByName(groupName); + for (const stateKey of groupStateKeys.get(groupName) ?? []) { + if (state[stateKey].groupSyncPending) { + state[stateKey].groupSyncPending = false; + groupStateChanged = true; + } + } + } catch (err: unknown) { + result.failed += 1; + for (const stateKey of groupStateKeys.get(groupName) ?? []) { + if (!state[stateKey].groupSyncPending) { + state[stateKey].groupSyncPending = true; + groupStateChanged = true; + } + } + logger.error(`[auto-sync] Group sync failed for ${groupName}: ${(err as Error).message}`); + } + } + if (groupStateChanged) await deps.saveState(state); + return result; +} + +function shortErrorMessage(err: unknown): string { + const message = err instanceof Error ? err.message : String(err); + return message.replace(/\s+/g, ' ').slice(0, 240); +} + +export function getConfiguredRepoPath( + project: Pick, + repoName: string, + remoteUrl?: string, +): string { + if (!remoteUrl) return path.resolve(project.localPath, repoName); + const identity = getAutoSyncRepoIdentity(remoteUrl); + return path.resolve(project.localPath, ...identity.split('/').slice(0, -1), repoName); +} + +export async function addRepoToGroup( + project: Pick, + groupPath: string, + registryName = groupPath, +): Promise { + if (!project.groupName) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName); + const config = await loadGroupConfig(groupDir); + if (config.repos[groupPath] === registryName) return false; + if (config.repos[groupPath] !== undefined) { + throw new Error(`group path ${groupPath} is already mapped to ${config.repos[groupPath]}`); + } + config.repos[groupPath] = registryName; + await writeGroupConfigAtomic(path.join(groupDir, 'group.yaml'), config); + return true; +} + +export function getAutoSyncRepoIdentity(remoteUrl: string): string { + validateAutoSyncRemoteUrl(remoteUrl); + const [, host, remotePath] = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim())!; + return `${host.toLowerCase()}/${remotePath.replace(/\.git$/i, '')}`; +} + +export async function syncGroupByName(groupName: string): Promise { + const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName); + const config = await loadGroupConfig(groupDir); + await syncGroup(config, { groupDir }); +} + +async function writeGroupConfigAtomic(filePath: string, config: unknown): Promise { + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); + await fs.rename(tmpPath, filePath); +} + +export function resolveActualConcurrency(configured: number, availableMemoryGB: number): number { + const memoryLimit = Math.max(1, Math.floor(availableMemoryGB / 2)); + return Math.max(1, Math.min(configured, memoryLimit)); +} + +async function buildWorkItems( + config: AutoSyncConfig, + deps: AutoSyncRunDeps, +): Promise { + const items: AutoSyncWorkItem[] = []; + const targetOwners = new Map(); + for (const project of config.projects) { + let cloneRoot: AutoSyncWorkItem['cloneRoot']; + try { + cloneRoot = await deps.resolveCloneRoot(project.localPath); + } catch (err: unknown) { + for (const remoteUrl of project.remoteUrls) { + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); + } + continue; + } + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName, remoteUrl); + const previous = targetOwners.get(targetDir); + if (previous !== undefined) { + throw new Error( + `Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`, + ); + } + targetOwners.set(targetDir, remoteUrl); + items.push({ project, remoteUrl, cloneRoot, repoName, targetDir }); + } catch (err: unknown) { + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); + } + } + } + return items; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + signal: AbortSignal | undefined, + worker: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let nextIndex = 0; + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (nextIndex < items.length) { + throwIfAborted(signal); + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await worker(items[currentIndex]); + throwIfAborted(signal); + } + }); + // Settle every runner before surfacing a failure. Promise.all rejects on the + // first error while siblings are still inside a clone or waiting on an + // analyze fork, and the caller treats that rejection as "the run is over" — + // it releases the watch mutex and exits, orphaning those children. Each + // runner already refuses new work at the abort check above, so waiting here + // costs nothing on the cancel path. + const settlements = await Promise.allSettled(runners); + const failure = settlements.find((s) => s.status === 'rejected'); + if (failure) throw (failure as PromiseRejectedResult).reason; + return results; +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new Error('Auto-sync run cancelled.'); +} + +interface AutoSyncWorkItem { + project: AutoSyncProjectConfig; + remoteUrl: string; + cloneRoot?: Awaited>; + repoName?: string; + targetDir?: string; + error?: string; +} + +async function syncFirstAvailableBranch(input: { + item: AutoSyncWorkItem; + repoName: string; + targetDir: string; + timeoutMs: number; + deps: AutoSyncRunDeps; + logger: AutoSyncLogger; +}): Promise< + | { ok: true; branch: string } + | { ok: false; status: 'branch_unavailable' | 'sync_timeout'; message: string } +> { + const failures: string[] = []; + let sawTimeout = false; + for (const branch of input.item.project.branches) { + try { + await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, { + allowedCloneRoot: input.item.cloneRoot!.root, + expectedRepoName: input.repoName, + quarantineRoot: input.item.cloneRoot!.quarantineRoot, + allowAutoSyncSsh: true, + timeoutMs: input.timeoutMs, + branch, + overwriteLocalChanges: input.item.project.overwriteLocalChanges, + }); + const currentBranch = await input.deps.getCurrentBranch(input.targetDir, input.timeoutMs); + if (currentBranch === branch) return { ok: true, branch }; + failures.push(`${branch}: checked out ${currentBranch ?? ''}`); + input.logger.warn( + `[auto-sync] Branch ${branch} for ${input.item.remoteUrl} synced but current branch is ${currentBranch ?? ''}; trying next branch.`, + ); + } catch (err: unknown) { + const message = (err as Error).message; + if (message.includes('timed out')) sawTimeout = true; + failures.push(`${branch}: ${message}`); + input.logger.warn( + `[auto-sync] Branch ${branch} unavailable for ${input.item.remoteUrl}: ${message}`, + ); + } + } + return { + ok: false, + status: sawTimeout ? 'sync_timeout' : 'branch_unavailable', + message: failures.join('; '), + }; +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts new file mode 100644 index 000000000..624e092ec --- /dev/null +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -0,0 +1,643 @@ +import fs from 'node:fs/promises'; +import crypto from 'node:crypto'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { isProcessAlive, readProcessStartTime } from '../../utils/process-identity.js'; +import { loadAutoSyncConfig } from './config.js'; +import { runAutoSyncOnce } from './runner.js'; +import { getAutoSyncMutexPath, getAutoSyncWatchDir } from './state.js'; + +export interface AutoSyncStartHandle { + stop(): Promise; +} + +export type WatchStatusState = + | 'running' + | 'cancelling' + | 'stopping' + | 'stopped' + | 'stale' + | 'error'; +export type AutoSyncWatchStopResult = 'stopped' | 'not_running' | 'refused' | 'timeout'; + +export interface WatchStatusRecord { + state: WatchStatusState; + pid?: number; + ownerId?: string; + configPath?: string; + message?: string; + updatedAt: string; +} + +export interface WatchOwnerRecord { + pid: number; + ownerId: string; + processStartTime: string; + createdAt: string; +} + +interface WatchStopRequestRecord { + pid: number; + ownerId: string; + processStartTime: string; + requestedAt: string; +} + +const WATCH_STOP_POLL_MS = 250; + +export interface AutoSyncWatchPaths { + pidPath: string; + mutexPath: string; + ownerPath: string; + statusPath: string; +} + +export interface AutoSyncWatchControlDeps { + isProcessAlive(pid: number): boolean; + readProcessCommand(pid: number): string | undefined; + readProcessStartTime(pid: number): string | undefined; + sleep(ms: number): Promise; +} + +export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWatchPaths { + const watchDir = getAutoSyncWatchDir(gitnexusDir); + return { + pidPath: path.join(watchDir, 'watch.pid'), + mutexPath: getAutoSyncMutexPath(gitnexusDir), + ownerPath: path.join(watchDir, 'watch.owner.json'), + statusPath: path.join(watchDir, 'watch.status.json'), + }; +} + +export async function startAutoSyncWatch( + options: { + setIntervalFn?: typeof setInterval; + clearIntervalFn?: typeof clearInterval; + runOnce?: typeof runAutoSyncOnce; + stderr?: Pick; + keepAlive?: boolean; + paths?: AutoSyncWatchPaths; + deps?: Partial; + } = {}, +): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const ownerId = crypto.randomUUID(); + const processStartTime = deps.readProcessStartTime(process.pid); + if (!processStartTime) { + stderr.write('[auto-sync] Unable to verify the watch process start time.\n'); + return null; + } + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + const releaseLock = await acquireWatchLock(paths, deps, stderr, processStartTime); + if (!releaseLock) return null; + + try { + await writeWatchOwner(paths, { + pid: process.pid, + ownerId, + processStartTime, + createdAt: new Date().toISOString(), + }); + await writeAtomicText(paths.pidPath, `${process.pid}\n`); + + const loaded = await loadAutoSyncConfig(); + if (loaded.ok === false) { + stderr.write(`${loaded.message}\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: process.pid, + ownerId, + message: loaded.message, + updatedAt: new Date().toISOString(), + }); + await cleanupWatchFiles(paths, ownerId, releaseLock); + return null; + } + await writeWatchStatus(paths, { + state: 'running', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + + const runOnce = options.runOnce ?? runAutoSyncOnce; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; + let activeRun: Promise | undefined; + let activeAbortController: AbortController | undefined; + let stopping = false; + let statusWrite = Promise.resolve(); + const updateStatus = (state: WatchStatusState, message?: string) => { + const write = statusWrite.then(() => + writeWatchStatus(paths, { + state, + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + message, + updatedAt: new Date().toISOString(), + }), + ); + statusWrite = write.catch(() => {}); + return write; + }; + const reportStatusWriteFailure = (error: unknown) => { + stderr.write(`[auto-sync] Failed to publish watch status: ${(error as Error).message}\n`); + }; + const runSafely = () => { + if (stopping) return; + if (activeRun) { + stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); + return; + } + const startedAt = new Date(); + stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); + const abortController = new AbortController(); + const run = runOnce(loaded.config, { + signal: abortController.signal, + onAnalysisCancellationRequested: () => { + if (!stopping) { + void updateStatus( + 'cancelling', + 'Analysis cancellation requested; waiting for the worker to reach a safe shutdown point.', + ).catch(reportStatusWriteFailure); + } + }, + }) + .then((result) => { + stderr.write( + `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, + ); + }) + .catch((err: unknown) => { + stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + stderr.write('[auto-sync] Watch loop finished: failed.\n'); + }) + .finally(async () => { + if (activeRun === run) { + activeRun = undefined; + activeAbortController = undefined; + } + if (!stopping) { + await updateStatus('running').catch(reportStatusWriteFailure); + } + }); + activeRun = run; + activeAbortController = abortController; + }; + + let stopPromise: Promise | undefined; + const stop = () => + (stopPromise ??= (async () => { + stopping = true; + clearIntervalFn(timer); + clearIntervalFn(controlTimer); + activeAbortController?.abort(); + try { + await updateStatus('stopping'); + await activeRun?.catch(() => {}); + await updateStatus('stopped'); + } finally { + await cleanupWatchFiles(paths, ownerId, releaseLock); + } + })()); + const checkStopRequest = async () => { + const request = await readStopRequest(stopRequestPath(paths, ownerId)); + if ( + request?.pid === process.pid && + request.ownerId === ownerId && + request.processStartTime === processStartTime + ) { + void stop().catch((error: unknown) => { + stderr.write(`[auto-sync] Failed to stop watch: ${(error as Error).message}\n`); + }); + } + }; + + runSafely(); + const controlTimer = setIntervalFn(() => void checkStopRequest(), WATCH_STOP_POLL_MS); + const timer = setIntervalFn(runSafely, loaded.config.syncIntervalMinutes * 60_000); + if (options.keepAlive === false) { + controlTimer.unref?.(); + timer.unref?.(); + } + return { stop }; + } catch (error) { + await cleanupWatchFiles(paths, ownerId, releaseLock).catch(() => {}); + throw error; + } +} + +async function acquireWatchLock( + paths: AutoSyncWatchPaths, + deps: AutoSyncWatchControlDeps, + stderr: Pick, + processStartTime: string, +): Promise<(() => Promise) | null> { + try { + return await acquireFileLock(paths.mutexPath, { + pid: process.pid, + processStartTime, + isProcessAlive: deps.isProcessAlive, + readProcessStartTime: deps.readProcessStartTime, + }); + } catch (err: unknown) { + if (!(err instanceof FileLockBusyError)) throw err; + } + + const owner = await readOwnerFile(paths.ownerPath); + if (!owner) { + stderr.write( + `[auto-sync] Watch mutex is held but owner metadata is not ready or invalid. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return null; + } + if (!deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch mutex remains after owner pid ${owner.pid} exited. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return null; + } + const reason = getWatchProcessIdentityError(owner, deps); + if (reason) { + stderr.write(`[auto-sync] Refusing to trust existing watch pid ${owner.pid}; ${reason}.\n`); + return null; + } + stderr.write(`[auto-sync] Watch is already running with pid ${owner.pid}.\n`); + return null; +} + +export async function stopAutoSyncWatch( + options: { + paths?: AutoSyncWatchPaths; + stderr?: Pick; + deps?: Partial; + timeoutMs?: number; + pollMs?: number; + } = {}, +): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const timeoutMs = options.timeoutMs ?? 10_000; + const pollMs = options.pollMs ?? 100; + const pid = await readPid(paths.pidPath); + if (!pid) { + const owner = await readOwnerFile(paths.ownerPath); + if (owner && deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch appears to be starting with pid ${owner.pid}; pid file is not ready.\n`, + ); + return 'refused'; + } + if (owner || (await fileExists(paths.mutexPath))) { + stderr.write( + `[auto-sync] Watch ownership is stale or incomplete. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; + } + stderr.write('[auto-sync] Watch is not running.\n'); + return 'not_running'; + } + if (!deps.isProcessAlive(pid)) { + stderr.write( + `[auto-sync] Watch pid ${pid} is stale. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; + } + + const owner = await readVerifiedWatchOwner(paths, pid, deps); + if (owner.ok === false) { + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; ${owner.reason}.\n`); + return 'refused'; + } + + const currentPid = await readPid(paths.pidPath); + const currentOwner = await readVerifiedWatchOwner(paths, pid, deps); + if ( + currentPid !== pid || + currentOwner.ok === false || + currentOwner.owner.ownerId !== owner.owner.ownerId + ) { + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; watch ownership changed.\n`); + return 'refused'; + } + + await writeAtomicText( + stopRequestPath(paths, owner.owner.ownerId), + `${JSON.stringify({ + pid, + ownerId: owner.owner.ownerId, + processStartTime: owner.owner.processStartTime, + requestedAt: new Date().toISOString(), + } satisfies WatchStopRequestRecord)}\n`, + ); + stderr.write(`[auto-sync] Stop requested for watch pid ${pid}.\n`); + const stopped = await waitForProcessExit(pid, { + deps, + timeoutMs, + pollMs, + processStartTime: owner.owner.processStartTime, + }); + if (!stopped) { + stderr.write(`[auto-sync] Watch pid ${pid} did not exit within ${timeoutMs}ms.\n`); + return 'timeout'; + } + return 'stopped'; +} + +export async function readAutoSyncWatchStatus( + paths = getAutoSyncWatchPaths(), + deps: Partial = {}, +): Promise { + const resolvedDeps = resolveWatchDeps(deps); + const pid = await readPid(paths.pidPath); + const stored = await readStatusFile(paths.statusPath); + const updatedAt = stored?.updatedAt ?? new Date().toISOString(); + if (pid && !resolvedDeps.isProcessAlive(pid)) { + return { + ...stored, + state: 'stale', + pid, + message: 'pid file exists but process is not running', + updatedAt, + }; + } + if (pid) { + const owner = await readVerifiedWatchOwner(paths, pid, resolvedDeps); + if (owner.ok === false) { + return { + ...stored, + state: 'error', + pid, + message: owner.reason, + updatedAt, + }; + } + if (stored?.state === 'error') { + return { + ...stored, + pid, + ownerId: owner.owner.ownerId, + updatedAt, + }; + } + return { + ...stored, + state: + stored?.state === 'cancelling' || stored?.state === 'stopping' ? stored.state : 'running', + pid, + ownerId: owner.owner.ownerId, + updatedAt, + }; + } + return stored ?? { state: 'stopped', updatedAt }; +} + +function isSafeWatchOwnerId(ownerId: string): boolean { + return ( + ownerId === path.basename(ownerId) && + !ownerId.includes('..') && + !ownerId.includes('/') && + !ownerId.includes('\\') + ); +} + +async function readOwnerFile(ownerPath: string): Promise { + try { + const raw = await fs.readFile(ownerPath, 'utf-8'); + const parsed = JSON.parse(raw) as WatchOwnerRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + isSafeWatchOwnerId(parsed.ownerId) && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime + ) { + return parsed; + } + return undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return undefined; + } +} + +async function readVerifiedWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, + deps: AutoSyncWatchControlDeps, +): Promise<{ ok: true; owner: WatchOwnerRecord } | { ok: false; reason: string }> { + const [status, owner] = await Promise.all([ + readStatusFile(paths.statusPath), + readOwnerFile(paths.ownerPath), + ]); + if (!owner) return { ok: false, reason: 'watch owner is missing or invalid' }; + if (!status) return { ok: false, reason: 'watch status is missing or invalid' }; + if (owner.pid !== pid) return { ok: false, reason: 'watch owner pid does not match pid file' }; + if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' }; + if (!status.ownerId || status.ownerId !== owner.ownerId) { + return { ok: false, reason: 'watch status owner does not match watch owner' }; + } + const identityError = getWatchProcessIdentityError(owner, deps); + if (identityError) return { ok: false, reason: identityError }; + return { ok: true, owner }; +} + +function getWatchProcessIdentityError( + owner: WatchOwnerRecord, + deps: AutoSyncWatchControlDeps, +): string | undefined { + const processStartTime = deps.readProcessStartTime(owner.pid); + if (!processStartTime) return 'unable to verify process start time'; + if (processStartTime !== owner.processStartTime) return 'pid belongs to a different process'; + const command = deps.readProcessCommand(owner.pid); + if (!command) return 'unable to verify process command'; + if ( + !/(?:^|\s)(?:watch|auto-sync)(?:\s|$)/.test(command) || + !/(?:gitnexus|[\\/]cli[\\/]index\.(?:ts|[cm]?js))/.test(command) + ) { + return 'pid command is not a GitNexus auto-sync process'; + } + return undefined; +} + +async function waitForProcessExit( + pid: number, + options: { + deps: AutoSyncWatchControlDeps; + timeoutMs: number; + pollMs: number; + processStartTime?: string; + }, +): Promise { + // A bare liveness poll cannot tell "still running" from "exited, and the OS + // handed the pid to something else" — so a reused pid would keep us waiting + // on an unrelated process and then report the watch stopped once THAT exits. + // The start time identifies the process behind the number. + const isOriginalProcessAlive = () => { + if (!options.deps.isProcessAlive(pid)) return false; + if (!options.processStartTime) return true; + const startTime = options.deps.readProcessStartTime(pid); + return startTime === undefined || startTime === options.processStartTime; + }; + const deadline = Date.now() + options.timeoutMs; + while (Date.now() < deadline) { + if (!isOriginalProcessAlive()) return true; + await options.deps.sleep(options.pollMs); + } + return !isOriginalProcessAlive(); +} + +async function readPid(pidPath: string): Promise { + try { + const raw = await fs.readFile(pidPath, 'utf-8'); + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 ? pid : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw err; + } +} + +async function readStatusFile(statusPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(statusPath, 'utf-8')) as WatchStatusRecord; + return parsed && typeof parsed === 'object' ? parsed : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return { + state: 'error', + message: `unable to read status file: ${(err as Error).message}`, + updatedAt: new Date().toISOString(), + }; + } +} + +function stopRequestPath(paths: AutoSyncWatchPaths, ownerId: string): string { + if (!isSafeWatchOwnerId(ownerId)) { + throw new Error('watch ownerId is not a safe filename component'); + } + return path.join(path.dirname(paths.pidPath), `watch.stop.${ownerId}.json`); +} + +async function readStopRequest(filePath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(filePath, 'utf-8')) as WatchStopRequestRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime && + typeof parsed.requestedAt === 'string' && + parsed.requestedAt + ) { + return parsed; + } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return undefined; + } + return undefined; +} + +async function writeWatchStatus( + paths: AutoSyncWatchPaths, + record: WatchStatusRecord, +): Promise { + await fs.mkdir(path.dirname(paths.statusPath), { recursive: true }); + const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, paths.statusPath); +} + +async function writeWatchOwner(paths: AutoSyncWatchPaths, record: WatchOwnerRecord): Promise { + await writeAtomicText(paths.ownerPath, `${JSON.stringify(record, null, 2)}\n`); +} + +async function cleanupWatchFiles( + paths: AutoSyncWatchPaths, + ownerId: string, + releaseLock: () => Promise, +): Promise { + try { + const owner = await readOwnerFile(paths.ownerPath); + if (owner?.ownerId === ownerId) { + if ((await readPid(paths.pidPath)) === owner.pid) await removeIfExists(paths.pidPath); + if ((await readOwnerFile(paths.ownerPath))?.ownerId === ownerId) { + await removeIfExists(paths.ownerPath); + } + await removeIfExists(stopRequestPath(paths, ownerId)); + } + } finally { + await releaseLock(); + } +} + +async function writeAtomicText(filePath: string, content: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, content, 'utf-8'); + await fs.rename(tmpPath, filePath); +} + +async function removeIfExists(filePath: string): Promise { + await fs.rm(filePath, { force: true }); +} + +async function fileExists(filePath: string): Promise { + return fs.access(filePath).then( + () => true, + () => false, + ); +} + +function resolveWatchDeps(deps: Partial = {}): AutoSyncWatchControlDeps { + return { + isProcessAlive: deps.isProcessAlive ?? isProcessAlive, + readProcessCommand: + deps.readProcessCommand ?? + ((pid) => { + try { + const command = + process.platform === 'win32' + ? execFileSync( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `(Get-CimInstance Win32_Process -Filter \"ProcessId = ${pid}\").CommandLine`, + ], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim() + : execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return command || undefined; + } catch { + return undefined; + } + }), + readProcessStartTime: deps.readProcessStartTime ?? readProcessStartTime, + sleep: + deps.sleep ?? + ((ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })), + }; +} diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts new file mode 100644 index 000000000..01f3ed2f5 --- /dev/null +++ b/gitnexus/src/core/auto-sync/state.ts @@ -0,0 +1,173 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped'; + +export interface AutoSyncCommitStateEntry { + codeCommitId: string; + analyzedCommitId?: string; + lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + analyzeConsecutiveFailures?: number; + lastAnalyzeError?: string; + groupSyncPending?: boolean; + lastSyncTime: string; +} + +export type AutoSyncCommitState = Record; + +export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'watch'); +} + +export function getAutoSyncMutexPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'watch.mutex'); +} + +export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json'); +} + +export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt'); +} + +export async function resetAutoSyncState(gitnexusDir = getGlobalDir()): Promise { + let releaseLock: () => Promise; + try { + releaseLock = await acquireFileLock(getAutoSyncMutexPath(gitnexusDir)); + } catch (error) { + if (error instanceof FileLockBusyError) return false; + throw error; + } + + try { + await Promise.all([ + fs.rm(getAutoSyncStatePath(gitnexusDir), { force: true }), + fs.rm(getProjectCommitInfoPath(gitnexusDir), { force: true }), + ]); + return true; + } finally { + await releaseLock(); + } +} + +export function buildStateKey(repoPath: string, branch: string): string { + return `${path.resolve(repoPath)}|${branch}`; +} + +export function shouldAnalyzeCommit(input: { + currentCommit: string; + previousAnalyzedCommit?: string; + previousStatus?: AutoSyncAnalyzeStatus; +}): boolean { + if (!input.currentCommit) return false; + if (input.previousStatus === 'failed') return true; + return input.currentCommit !== input.previousAnalyzedCommit; +} + +export async function loadAutoSyncState( + statePath = getAutoSyncStatePath(), +): Promise { + try { + const raw = await fs.readFile(statePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}; + return Object.fromEntries( + Object.entries(parsed).filter((entry): entry is [string, AutoSyncCommitStateEntry] => + isAutoSyncCommitStateEntry(entry[1]), + ), + ); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return {}; + // Corrupt JSON is genuinely unrecoverable, so rebuilding is the only move. + // An unreadable file (EACCES, EIO, EISDIR) is different: the state is + // probably intact, and returning {} here would make the tick overwrite it, + // losing every repo's analyzed commit and failure count. + if (!(err instanceof SyntaxError)) throw err; + process.stderr.write( + `[auto-sync] Ignoring corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + return {}; + } +} + +function isAutoSyncCommitStateEntry(value: unknown): value is AutoSyncCommitStateEntry { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const entry = value as Record; + return ( + typeof entry.codeCommitId === 'string' && + typeof entry.lastSyncTime === 'string' && + (entry.analyzedCommitId === undefined || typeof entry.analyzedCommitId === 'string') && + (entry.lastAnalyzeStatus === undefined || + entry.lastAnalyzeStatus === 'success' || + entry.lastAnalyzeStatus === 'failed' || + entry.lastAnalyzeStatus === 'skipped' || + entry.lastAnalyzeStatus === 'threshold_skipped') && + (entry.analyzeConsecutiveFailures === undefined || + (typeof entry.analyzeConsecutiveFailures === 'number' && + Number.isInteger(entry.analyzeConsecutiveFailures) && + entry.analyzeConsecutiveFailures >= 0)) && + (entry.lastAnalyzeError === undefined || typeof entry.lastAnalyzeError === 'string') && + (entry.groupSyncPending === undefined || typeof entry.groupSyncPending === 'boolean') + ); +} + +export async function saveAutoSyncState( + state: AutoSyncCommitState, + statePath = getAutoSyncStatePath(), +): Promise { + await fs.mkdir(path.dirname(statePath), { recursive: true }); + const tmpPath = `${statePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, statePath); +} + +export async function writeProjectCommitInfo( + entries: ProjectCommitInfoEntry[], + infoPath = getProjectCommitInfoPath(), +): Promise { + await fs.mkdir(path.dirname(infoPath), { recursive: true }); + const lines = [ + '# GitNexus auto-sync project commit info', + `updated_at: ${new Date().toISOString()}`, + '', + ...entries.flatMap((entry) => [ + `remote: ${entry.remoteUrl}`, + `local_path: ${entry.localPath}`, + `branch: ${entry.branch ?? ''}`, + `code_commit: ${entry.codeCommitId ?? ''}`, + `analyzed_commit: ${entry.analyzedCommitId ?? ''}`, + `status: ${entry.status}`, + `analyze_consecutive_failures: ${entry.analyzeConsecutiveFailures ?? 0}`, + ...(entry.analyzeFailureThreshold === undefined + ? [] + : [`analyze_failure_threshold: ${entry.analyzeFailureThreshold}`]), + ...(entry.lastAnalyzeError ? [`last_analyze_error: ${entry.lastAnalyzeError}`] : []), + `last_sync_time: ${entry.lastSyncTime}`, + '', + ]), + ]; + const tmpPath = `${infoPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${lines.join('\n')}\n`, 'utf-8'); + await fs.rename(tmpPath, infoPath); +} + +export interface ProjectCommitInfoEntry { + remoteUrl: string; + localPath: string; + branch?: string; + codeCommitId?: string; + analyzedCommitId?: string; + status: + | AutoSyncAnalyzeStatus + | 'sync_failed' + | 'branch_skipped' + | 'branch_unavailable' + | 'sync_timeout'; + analyzeConsecutiveFailures?: number; + analyzeFailureThreshold?: number; + lastAnalyzeError?: string; + lastSyncTime: string; +} diff --git a/gitnexus/src/core/group/group-lock.ts b/gitnexus/src/core/group/group-lock.ts index 7d25753af..cdaacb283 100644 --- a/gitnexus/src/core/group/group-lock.ts +++ b/gitnexus/src/core/group/group-lock.ts @@ -21,12 +21,11 @@ * else claims: group directories live under `~/.gitnexus/groups/` (or * `$GITNEXUS_HOME`), never under a repo's `.gitnexus[/branches/]`. * - * WHY IT FAILS CLOSED, unlike the registry lock. `withRegistryLock` degrades to - * running UNLOCKED on timeout, and that is right for it: it guards a sub-second - * JSON read/merge/write on a latency-critical path (`augment` runs on every - * editor tool call), and running unlocked is merely the pre-lock status quo. A - * group sync is the opposite on every axis — it is long, expensive, operator- - * initiated, and its lost update destroys contracts rather than a registry field. + * WHY IT FAILS CLOSED, like the registry lock. `withRegistryLock` also + * refuses to continue unlocked on timeout: a lost registry update can drop a + * concurrent registration. A group sync still fails closed for additional + * reasons — it is long, expensive, operator-initiated, and a lost update + * destroys contracts rather than a registry field. * A sync that cannot be protected must not run at all, and there are three * distinct ways it can fail to be protected; all three throw * {@link GroupSyncLockError}: diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index eadee1ce0..40ccdb8e0 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -72,6 +72,7 @@ import { shadowSidecarRecoveryMessage, sidecarPreflightDisabled, } from './sidecar-recovery.js'; +import { isProcessAlive } from '../../utils/process-identity.js'; import { logger } from '../logger.js'; import { @@ -330,20 +331,6 @@ const INIT_LOCK_RETRY_DELAY_MS = 500; const initLockPath = (dbPath: string): string => `${dbPath}.init.lock`; -/** - * Returns true when the process identified by `pid` is still running. - * Uses `process.kill(pid, 0)` which sends signal 0 (a no-op probe) — - * it throws ESRCH when the process does not exist. - */ -const isProcessAlive = (pid: number): boolean => { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -}; - /** * Try to break a stale lock whose owning process has exited. * Returns `true` if the stale lock was removed (caller should retry acquire). diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index 81193598e..d573f5a5c 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -26,13 +26,20 @@ import type { AnalyzeOptions } from '../core/run-analyze.js'; import type { AnalyzeResultIpc } from './analyze-worker-ipc.js'; -/** Parent → child: the single command that starts an analysis run. */ +/** Parent → child: start one analysis run. */ export interface StartMessage { type: 'start'; repoPath: string; options: AnalyzeOptions; } +/** Parent → child: request safe cancellation at the next JS-visible checkpoint. */ +export interface CancelMessage { + type: 'cancel'; +} + +export type ParentMessage = StartMessage | CancelMessage; + export interface ProgressMessage { type: 'progress'; phase: string; diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index ad4632eb0..d41a4b34d 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -6,12 +6,13 @@ * * IPC Protocol: * Parent -> Child: { type: 'start', repoPath: string, options: AnalyzeOptions } + * Parent -> Child: { type: 'cancel' } * Child -> Parent: { type: 'progress', phase: string, percent: number, message: string } * Child -> Parent: { type: 'complete', result: AnalyzeResult } * Child -> Parent: { type: 'error', message: string } */ -import type { StartMessage, WorkerMessage } from './analyze-worker-protocol.js'; +import type { ParentMessage, WorkerMessage } from './analyze-worker-protocol.js'; import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js'; type BoundedCheckpointBeforeExit = typeof import('../core/lbug/shutdown-helpers.js').boundedCheckpointBeforeExit; @@ -62,19 +63,24 @@ process.on('unhandledRejection', (reason: unknown) => { } }); -// Handle cancellation / timeout shutdown (analyze-job.ts `cancelJob` sends -// SIGTERM). Bounded CHECKPOINT-then-exit shared with the CLI SIGINT path (#2264): -// skip the native close (the LadybugDB destructor can double-free after --pdg -// writes), but don't block behind the in-flight COPY's connection lock — so a -// single cancel can't abort or hang the worker. A CHECKPOINT failure is reported -// to the parent over IPC, not swallowed; the exit always fires. -process.on('SIGTERM', () => { - // Only report the cancellation if the analysis hasn't already reported a - // terminal outcome (#2264 P3) — otherwise this would flip an already-complete - // job to failed. The cleanup + exit below run regardless. +// IPC cancellation is the cross-platform control path. It only records the +// request while analysis is active; cleanup waits until the analysis promise has +// returned to JS. SIGTERM is retained only for local process shutdown. +let cancellationRequested = false; +let started = false; +function requestWorkerCancellation(source: string): void { + if (cancellationRequested) return; + cancellationRequested = true; if (claimTerminal()) { - send({ type: 'error', message: 'Analysis cancelled (worker received SIGTERM)' }); + send({ type: 'error', message: `Analysis cancelled (${source})` }); } + if (!started) { + // No analysis has started, so no native work needs a safe-point handshake. + process.exit(0); + } +} + +function exitAfterCancellation(): void { if (!boundedCheckpointBeforeExit) { process.exit(0); return; @@ -83,16 +89,21 @@ process.on('SIGTERM', () => { exitCode: 0, onFlushError: (err: unknown) => { const message = - err instanceof Error ? err.message : 'Worker checkpoint failed during SIGTERM'; + err instanceof Error ? err.message : 'Worker checkpoint failed during cancellation'; send({ type: 'error', message }); }, }); -}); +} -// Listen for start command from parent — guarded against re-entry -let started = false; -process.on('message', async (msg: StartMessage) => { - if (msg.type !== 'start' || started) return; +process.on('SIGTERM', () => requestWorkerCancellation('worker received SIGTERM')); + +// Listen for parent commands — guarded against re-entry. +process.on('message', async (msg: ParentMessage) => { + if (msg.type === 'cancel') { + requestWorkerCancellation('parent requested cancellation'); + return; + } + if (started) return; started = true; try { @@ -112,6 +123,9 @@ process.on('message', async (msg: StartMessage) => { }, ); boundedCheckpointBeforeExit = prepared.loaded.shutdownHelpers.boundedCheckpointBeforeExit; + // A cancel can arrive while the dynamic imports are resolving. Do not begin + // a new analysis after that request; the finally block performs safe cleanup. + if (cancellationRequested) return; // The run → finalize → report contract lives in the side-effect-free // analyze-worker-core seam (unit-testable without this entry module's // process.on side effects). It reports exactly one terminal message and @@ -135,9 +149,11 @@ process.on('message', async (msg: StartMessage) => { }); } } finally { - // LadybugDB's native module prevents clean exit — force it (same reason the - // CLI uses process.exit(0)). In `finally` so the exit still fires even if the - // report above throws on a closed IPC channel (#2264 review P3). - setTimeout(() => process.exit(0), 500); + // A cancel must not end the process while runFullAnalysis may still be in + // native code. This continuation runs only after that promise has settled. + if (cancellationRequested) exitAfterCancellation(); + // Normal terminal outcomes still need the existing process exit because + // LadybugDB stays live. + else setTimeout(() => process.exit(0), 500); } }); diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 2758391aa..fc3c1300a 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -58,7 +58,7 @@ import { assertString, BadRequestError, createRouteLimiter } from './validation. import { parseGrepQuery, GREP_TIME_BUDGET_MS } from './grep-params.js'; import { runGrepScanInWorker } from './grep-scan.js'; import { - extractRepoName, + extractWebRepoName, getCloneDir, cloneOrPull, warnIfInsecureAzureConfig, @@ -1598,7 +1598,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => try { // Clone if URL provided if (repoUrl && !repoLocalPath) { - const repoName = extractRepoName(repoUrl); + const repoName = extractWebRepoName(repoUrl); targetPath = getCloneDir(repoName); jobManager.updateJob(job.id, { diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index 742be319c..1cd204761 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -9,9 +9,15 @@ import { spawn } from 'child_process'; import path from 'path'; import fs from 'fs/promises'; import { isIP } from 'net'; +import os from 'node:os'; import { logger } from '../core/logger.js'; -import { parseRepoNameFromUrl, stripUrlCredentials } from '../storage/git.js'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { sanitizeRepoName, stripUrlCredentials } from '../storage/git.js'; +import { + assertDirectoryOwnerAndPermissions, + quarantineAutoSyncPartial, +} from '../core/auto-sync/path-security.js'; +import { validateAutoSyncRemoteUrl } from '../core/auto-sync/config.js'; /** * Root directory for all cloned repositories. Targets must resolve inside this. @@ -39,12 +45,16 @@ export const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; * clone root via path traversal. */ export function extractRepoName(url: string): string { - const name = parseRepoNameFromUrl(url); + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const name = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; if ( !name || name === '.' || name === '..' || name === 'unknown' || + name.startsWith('-') || !REPO_NAME_PATTERN.test(name) ) { throw new Error('Could not extract a valid repository name from URL'); @@ -52,6 +62,26 @@ export function extractRepoName(url: string): string { return name; } +/** + * Derive a clone directory name for the web `/api/analyze` boundary. + * + * The API historically accepted Azure DevOps and similar URLs whose repo + * segment contains spaces or other directory-unsafe characters by sanitizing + * the final segment. Keep that compatibility at the web boundary while leaving + * `extractRepoName()` strict for internal/security-sensitive callers. + */ +export function extractWebRepoName(url: string): string { + let trimmed = url.trim(); + while (trimmed.endsWith('/')) trimmed = trimmed.slice(0, -1); + const withoutGit = trimmed.toLowerCase().endsWith('.git') ? trimmed.slice(0, -4) : trimmed; + const rawName = withoutGit.split(/[/:]/).filter(Boolean).pop() ?? ''; + const safeName = sanitizeRepoName(rawName); + if (!rawName || safeName === 'unknown') { + throw new Error('Could not extract a valid repository name from URL'); + } + return safeName; +} + /** Get the clone target directory for a repo name. */ export function getCloneDir(repoName: string): string { // Re-validate at the boundary even though extractRepoName already checked — @@ -87,6 +117,10 @@ export function validateGitUrl(url: string): void { throw new Error('Only https:// and http:// git URLs are allowed'); } + if (parsed.search || parsed.hash) { + throw new Error('Git URLs must not include query strings or fragments'); + } + const host = parsed.hostname.toLowerCase(); // Block known dangerous hostnames (cloud metadata services) @@ -235,6 +269,26 @@ export interface CloneProgress { message: string; } +export interface CloneOrPullOptions { + token?: string; + allowedCloneRoot?: string; + expectedRepoName?: string; + quarantineRoot?: string; + allowAutoSyncSsh?: boolean; + timeoutMs?: number; + branch?: string; + overwriteLocalChanges?: boolean; + runGitForTest?: typeof runGit; +} + +type RunGitOptions = { + token?: string; + url?: string; + timeoutMs?: number; + timeoutKillGraceMs?: number; + spawnForTest?: typeof spawn; +}; + /** * Build the `git clone` argument list for a given URL and target directory. * @@ -304,6 +358,10 @@ export function buildCloneArgs(url: string, targetDir: string): string[] { return ['clone', '--depth', '1', '--', url, targetDir]; } +export function buildBranchCloneArgs(url: string, targetDir: string, branch: string): string[] { + return ['clone', '--depth', '1', '--branch', branch, '--', url, targetDir]; +} + /** * Normalize a git URL into a comparable form. * @@ -363,27 +421,14 @@ export function normalizeGitUrlForCompare(url: string): string { * remote means for its threat model — for cloneOrPull, a missing remote * on an existing clone is treated as a refuse-to-pull condition. */ -export function getRemoteOriginUrl(cwd: string): Promise { - return new Promise((resolve) => { - const proc = spawn('git', ['config', '--get', 'remote.origin.url'], { - cwd, - stdio: ['ignore', 'pipe', 'pipe'], - windowsHide: true, - env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, - }); - let stdout = ''; - proc.stdout.on('data', (chunk: Buffer) => { - stdout += chunk; - }); - proc.on('close', (code) => { - if (code === 0 && stdout.trim()) { - resolve(stdout.trim()); - } else { - resolve(null); - } - }); - proc.on('error', () => resolve(null)); - }); +export async function getRemoteOriginUrl(cwd: string, timeoutMs?: number): Promise { + try { + const stdout = await runGit(['config', '--get', 'remote.origin.url'], cwd, { timeoutMs }); + return stdout.trim() || null; + } catch (error) { + if ((error as Error).message.includes('timed out')) throw error; + return null; + } } /** @@ -403,8 +448,9 @@ export function getRemoteOriginUrl(cwd: string): Promise { export async function assertRemoteMatchesRequestedUrl( targetDir: string, requestedUrl: string, + timeoutMs?: number, ): Promise { - const remoteUrl = await getRemoteOriginUrl(targetDir); + const remoteUrl = await getRemoteOriginUrl(targetDir, timeoutMs); if (remoteUrl === null) { throw new Error(`Existing clone at ${targetDir} has no remote.origin — refusing to pull`); } @@ -446,51 +492,213 @@ export async function cloneOrPull( url: string, targetDir: string, onProgress?: (progress: CloneProgress) => void, - options?: { token?: string }, + options?: CloneOrPullOptions, ): Promise { // Containment barrier — inline with the canonical path.relative idiom so // CodeQL recognizes the sanitizer at every following filesystem and // subprocess sink. The same `safeTarget` is used for every downstream // path operation — no reassignment that the analyzer could lose track of. // - // Limitation: this is a lexical containment check, not a realpath check. - // If an attacker can place a symlink under CLONE_ROOT pointing outside it, - // the lexical check passes but the clone lands at the symlink target. That - // requires pre-existing local write access to CLONE_ROOT, so the threat - // model considers it out of scope; CodeQL js/path-injection accepts the - // lexical form. Tracked as a follow-up if defense-in-depth is needed. + // The lexical check runs before filesystem creation; realpath and symlink + // checks below run before pull/clone and again after clone completes. + const cloneRoot = path.resolve(options?.allowedCloneRoot ?? CLONE_ROOT); + const expectedRepoName = options?.expectedRepoName; + if (expectedRepoName !== undefined && expectedRepoName !== extractRepoName(url)) { + throw new Error(`Clone target repo name ${expectedRepoName} does not match requested URL`); + } + const safeTarget = path.resolve(targetDir); - const rel = path.relative(CLONE_ROOT, safeTarget); + if (expectedRepoName !== undefined && path.basename(safeTarget) !== expectedRepoName) { + throw new Error(`Clone target basename must match repository name ${expectedRepoName}`); + } + + const rel = path.relative(cloneRoot, safeTarget); if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { - throw new Error(`Clone target must be a subdirectory of ${CLONE_ROOT}`); + throw new Error(`Clone target must be a subdirectory of ${cloneRoot}`); } // Always validate the requested URL — the prior shape only ran this in // the code path where the repo was cloned. Now it runs unconditionally, // preventing SSRF / blocked-host bypasses even when targetDir already exists. - validateGitUrl(url); + if (options?.allowAutoSyncSsh) validateAutoSyncRemoteUrl(url); + else validateGitUrl(url); + await fs.mkdir(cloneRoot, { recursive: true }); + if (options?.allowedCloneRoot) { + await assertDirectoryOwnerAndPermissions(cloneRoot); + } + await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); + await fs.mkdir(path.dirname(safeTarget), { recursive: true }); + await assertNoSymlinkPath(cloneRoot, safeTarget, Boolean(options?.allowedCloneRoot)); + await assertPreRealpathContainment(cloneRoot, safeTarget); const exists = await fs.access(path.join(safeTarget, '.git')).then( () => true, () => false, ); + const targetExists = await fs.access(safeTarget).then( + () => true, + () => false, + ); + if (exists) { + if (options?.allowedCloneRoot) { + await assertNoSymlinkPath(cloneRoot, path.join(safeTarget, '.git'), true); + } + await assertPostRealpathContainment(cloneRoot, safeTarget); // Confirm the existing clone is actually the same repository the caller // requested. Without this check, a pull would silently succeed against // whatever remote the dir was originally cloned from. - await assertRemoteMatchesRequestedUrl(safeTarget, url); + await assertRemoteMatchesRequestedUrl(safeTarget, url, options?.timeoutMs); onProgress?.({ phase: 'pulling', message: 'Pulling latest changes...' }); - await runGit(['pull', '--ff-only'], safeTarget, { token: options?.token, url }); + const runGitImpl = options?.runGitForTest ?? runGit; + if (options?.branch) { + if (!options.overwriteLocalChanges) { + const status = await runGitImpl(['status', '--porcelain'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + if (status.trim()) { + throw new Error( + `Refusing to update ${safeTarget}: local changes detected. Set overwrite_local_changes: true to overwrite them.`, + ); + } + } + await runGitImpl( + [ + 'fetch', + '--depth', + '1', + 'origin', + `refs/heads/${options.branch}:refs/remotes/origin/${options.branch}`, + ], + safeTarget, + { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }, + ); + await runGitImpl( + [ + 'checkout', + ...(options.overwriteLocalChanges ? ['--force'] : []), + '-B', + options.branch, + `origin/${options.branch}`, + ], + safeTarget, + { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }, + ); + if (options.overwriteLocalChanges) { + // `checkout --force` rewrites tracked files only, so untracked sources + // left by an operator or an earlier branch survive and then get indexed + // as if they were part of the remote commit. Deliberately no `-x`/`-X`: + // ignored paths must survive, and `-e /.gitnexus` is belt-and-braces + // because `.git/info/exclude` is skipped on a read-only storage mount + // and a freshly cloned repo may not have been analyzed yet at all. + await runGitImpl(['clean', '--force', '-d', '-e', '/.gitnexus'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + } + } else { + await runGitImpl(['pull', '--ff-only'], safeTarget, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + } } else { - await fs.mkdir(path.dirname(safeTarget), { recursive: true }); + if (targetExists && (await fs.readdir(safeTarget)).length > 0) { + throw new Error(`Clone target already exists but is not a git repository: ${safeTarget}`); + } onProgress?.({ phase: 'cloning', message: `Cloning ${url}...` }); - await runGit(buildCloneArgs(url, safeTarget), undefined, { token: options?.token, url }); + try { + const runGitImpl = options?.runGitForTest ?? runGit; + const cloneArgs = options?.branch + ? buildBranchCloneArgs(url, safeTarget, options.branch) + : buildCloneArgs(url, safeTarget); + await runGitImpl(cloneArgs, undefined, { + token: options?.token, + url, + timeoutMs: options?.timeoutMs, + }); + await assertPostRealpathContainment(cloneRoot, safeTarget); + } catch (err: unknown) { + if (options?.quarantineRoot) { + const partialExists = await fs.access(safeTarget).then( + () => true, + () => false, + ); + if (partialExists) { + try { + await quarantineAutoSyncPartial(safeTarget, options.quarantineRoot); + } catch (quarantineError) { + throw new AggregateError( + [err, quarantineError], + `Clone failed and partial checkout could not be quarantined: ${safeTarget}`, + ); + } + } + } + throw err; + } } return safeTarget; } +async function assertPreRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realParent = await fs.realpath(path.dirname(target)); + const parentRel = path.relative(realRoot, realParent); + if (parentRel.startsWith('..') || path.isAbsolute(parentRel)) { + throw new Error(`Clone target parent must resolve inside ${root}`); + } +} + +async function assertPostRealpathContainment(root: string, target: string): Promise { + const realRoot = await fs.realpath(root); + const realTarget = await fs.realpath(target); + const rel = path.relative(realRoot, realTarget); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { + throw new Error(`Clone target must resolve inside ${root}`); + } +} + +async function assertNoSymlinkPath( + root: string, + target: string, + verifyOwnership = false, +): Promise { + const resolvedRoot = path.resolve(root); + const resolvedTarget = path.resolve(target); + const relativeTarget = path.relative(resolvedRoot, resolvedTarget); + if (relativeTarget.startsWith('..') || path.isAbsolute(relativeTarget)) return; + let current = resolvedRoot; + for (const segment of relativeTarget.split(path.sep).filter(Boolean)) { + current = path.join(current, segment); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlink in clone target path: ${current}`); + } + if (verifyOwnership) await assertDirectoryOwnerAndPermissions(current); + } +} + /** * Hosts the per-request GitHub PAT may be sent to. Exported so the * /api/analyze boundary check and this injection-site check share one @@ -592,11 +800,10 @@ function warnIfCleartextCredential(url?: string): void { } /** - * Build the spawn env for `git`. Suppresses credential prompts and, when a - * credential resolves (see resolveGitCredential), injects a single - * host-scoped Authorization header via the `GIT_CONFIG_*` env protocol - * (git ≥2.31) so credentials never appear in argv or the URL. Appends after - * any existing `GIT_CONFIG_COUNT` rather than overwriting it. Exported for + * Build the spawn env for managed `git` commands. Suppresses credential + * prompts, disables repository hooks, and injects at most one host-scoped + * Authorization header via the `GIT_CONFIG_*` env protocol (git ≥2.31). + * Managed settings append after any existing GIT_CONFIG_COUNT. Exported for * unit tests. */ export function buildGitEnv( @@ -619,18 +826,21 @@ export function buildGitEnv( GIT_CURL_VERBOSE: undefined, }; + const existing = Number.parseInt(env.GIT_CONFIG_COUNT ?? '', 10); + let next = Number.isInteger(existing) && existing > 0 ? existing : 0; + env[`GIT_CONFIG_KEY_${next}`] = 'core.hooksPath'; + env[`GIT_CONFIG_VALUE_${next}`] = os.devNull; + next += 1; + const credential = resolveGitCredential(options); const key = options?.url ? buildExtraHeaderKey(options.url) : undefined; if (credential && key) { - // Append after any GIT_CONFIG_* the operator already set, so we never - // clobber their git config (e.g. an enforced http.sslVerify). - const existing = Number.parseInt(env.GIT_CONFIG_COUNT ?? '', 10); - const base = Number.isInteger(existing) && existing > 0 ? existing : 0; - env.GIT_CONFIG_COUNT = String(base + 1); - env[`GIT_CONFIG_KEY_${base}`] = key; - env[`GIT_CONFIG_VALUE_${base}`] = `Authorization: Basic ${credential}`; + env[`GIT_CONFIG_KEY_${next}`] = key; + env[`GIT_CONFIG_VALUE_${next}`] = `Authorization: Basic ${credential}`; + next += 1; warnIfCleartextCredential(options?.url); } + env.GIT_CONFIG_COUNT = String(next); return env; } @@ -640,35 +850,65 @@ export function buildGitEnv( // host-scoped Authorization header (GitHub PAT for github.com, else the // server's AZURE_DEVOPS_PAT for Azure hosts) via the GIT_CONFIG_* protocol — // never in argv. See resolveGitCredential / buildExtraHeaderKey. -function runGit( - args: string[], - cwd?: string, - options?: { token?: string; url?: string }, -): Promise { +export function runGit(args: string[], cwd?: string, options?: RunGitOptions): Promise { return new Promise((resolve, reject) => { - const proc = spawn('git', args, { + const spawnGit = options?.spawnForTest ?? spawn; + const proc = spawnGit('git', args, { cwd, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, env: buildGitEnv(process.env, options), }); + let stdout = ''; let stderr = ''; + let settled = false; + let timedOut = false; + let killTimer: NodeJS.Timeout | undefined; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + fn(); + }; + const timer = + options?.timeoutMs && options.timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + proc.kill('SIGTERM'); + killTimer = setTimeout(() => { + proc.kill('SIGKILL'); + finish(() => + reject(new Error(`git ${args[0]} timed out after ${options.timeoutMs}ms`)), + ); + }, options.timeoutKillGraceMs ?? 1_000); + }, options.timeoutMs) + : undefined; + proc.stdout?.on('data', (chunk: Buffer) => { + stdout += chunk; + }); proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk; }); proc.on('close', (code) => { - if (code === 0) resolve(); + if (timedOut) { + finish(() => reject(new Error(`git ${args[0]} timed out after ${options?.timeoutMs}ms`))); + return; + } + if (code === 0) finish(() => resolve(stdout)); else { // Log full stderr internally but don't expose it to API callers (SSRF mitigation) if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`); - reject(new Error(`git ${args[0]} failed (exit code ${code})`)); + finish(() => reject(new Error(`git ${args[0]} failed (exit code ${code})`))); } }); proc.on('error', (err) => { - reject(new Error(`Failed to spawn git: ${err.message}`)); + finish(() => reject(new Error(`Failed to spawn git: ${err.message}`))); }); }); } + +export const runGitForTest = runGit; diff --git a/gitnexus/src/storage/file-lock.ts b/gitnexus/src/storage/file-lock.ts new file mode 100644 index 000000000..a857d6a47 --- /dev/null +++ b/gitnexus/src/storage/file-lock.ts @@ -0,0 +1,189 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { isProcessAlive, readProcessStartTime } from '../utils/process-identity.js'; + +const HOSTNAME = os.hostname(); + +export interface FileLockOptions { + retries?: number; + retryDelayMs?: number; + pid?: number; + processStartTime?: string; + hostname?: string; + isProcessAlive?: (pid: number) => boolean; + readProcessStartTime?: (pid: number) => string | undefined; +} + +interface FileLockOwner { + pid: number; + ownerId: string; + processStartTime: string; + hostname: string; +} + +export class FileLockBusyError extends Error { + constructor(public readonly lockPath: string) { + super( + `Lock is already held: ${lockPath}. Confirm no owner process is active, then remove it manually.`, + ); + this.name = 'FileLockBusyError'; + } +} + +/** Acquire a recoverable cross-process mutex using an atomically published owner file. */ +export async function acquireFileLock( + lockPath: string, + options: FileLockOptions = {}, +): Promise<() => Promise> { + const resolvedPath = path.resolve(lockPath); + const retries = options.retries ?? 0; + const retryDelayMs = options.retryDelayMs ?? 50; + const pid = options.pid ?? process.pid; + const owner: FileLockOwner = { + pid, + ownerId: crypto.randomUUID(), + processStartTime: + options.processStartTime ?? (options.readProcessStartTime ?? readProcessStartTime)(pid) ?? '', + hostname: options.hostname ?? HOSTNAME, + }; + if (!owner.processStartTime) { + throw new Error(`Unable to determine process start time for file lock owner pid ${owner.pid}.`); + } + + await fs.mkdir(path.dirname(resolvedPath), { recursive: true }); + const pendingPath = `${resolvedPath}.pending-${owner.ownerId}`; + await fs.writeFile(pendingPath, `${JSON.stringify(owner)}\n`, { encoding: 'utf-8', flag: 'wx' }); + + try { + for (let attempt = 0; ; attempt += 1) { + try { + await fs.link(pendingPath, resolvedPath); + break; + } catch (error) { + if (!(await isLockConflict(error, resolvedPath))) throw error; + if ( + await reclaimStaleLock( + resolvedPath, + owner, + options.isProcessAlive ?? isProcessAlive, + options.readProcessStartTime ?? readProcessStartTime, + ) + ) { + continue; + } + if (attempt >= retries) throw new FileLockBusyError(lockPath); + await sleep(retryDelayMs); + } + } + } finally { + // The lock is already published by now, but the release closure below is + // not yet in the caller's hands. Letting a staging-file cleanup error + // escape would strand a lock nobody can release, so prefer leaking the + // pending file — its name is per-acquisition, so it can never block anyone. + await fs.rm(pendingPath, { force: true }).catch(() => {}); + } + + let releasePromise: Promise | undefined; + return () => (releasePromise ??= releaseOwnedLock(resolvedPath, owner.ownerId)); +} + +async function reclaimStaleLock( + lockPath: string, + guardOwner: FileLockOwner, + ownerIsAlive: (pid: number) => boolean, + getProcessStartTime: (pid: number) => string | undefined, +): Promise { + const reclaimGuardPath = `${lockPath}.reclaim`; + let releaseReclaimGuard: () => Promise; + try { + releaseReclaimGuard = await acquireFileLock(reclaimGuardPath, { + pid: guardOwner.pid, + processStartTime: guardOwner.processStartTime, + hostname: guardOwner.hostname, + isProcessAlive: ownerIsAlive, + readProcessStartTime: getProcessStartTime, + }); + } catch (error) { + if (error instanceof FileLockBusyError) return false; + throw error; + } + + try { + const owner = await readOwner(lockPath); + if (!owner) return false; + // A pid only means something on the machine that issued it. Asking this + // kernel about a holder on another host answers about an unrelated process + // — or nothing — and either way the answer is "stale", which would steal a + // live lock whenever GITNEXUS_HOME is a shared volume. + if (owner.hostname !== guardOwner.hostname) return false; + if (ownerIsAlive(owner.pid)) { + const currentStartTime = getProcessStartTime(owner.pid); + if (!currentStartTime || currentStartTime === owner.processStartTime) return false; + } + + await fs.rm(lockPath, { force: true }); + return true; + } finally { + await releaseReclaimGuard(); + } +} + +async function releaseOwnedLock(lockPath: string, ownerId: string): Promise { + const releasePath = `${lockPath}.release-${ownerId}-${crypto.randomUUID()}`; + if (!(await moveOwnedLock(lockPath, releasePath, ownerId))) return; + await fs.rm(releasePath, { force: true }); +} + +async function moveOwnedLock( + lockPath: string, + destinationPath: string, + ownerId: string, +): Promise { + if ((await readOwner(lockPath))?.ownerId !== ownerId) return false; + try { + await fs.rename(lockPath, destinationPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } + + if ((await readOwner(destinationPath))?.ownerId === ownerId) return true; + await fs.rename(destinationPath, lockPath).catch(() => {}); + return false; +} + +async function readOwner(lockPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(lockPath, 'utf-8')) as Partial; + if ( + Number.isInteger(parsed.pid) && + Number(parsed.pid) > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime && + typeof parsed.hostname === 'string' && + parsed.hostname + ) { + return parsed as FileLockOwner; + } + } catch { + // Invalid or legacy locks fail closed; only verified dead owners are reclaimed. + } + return undefined; +} + +async function isLockConflict(error: unknown, lockPath: string): Promise { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'EEXIST') return true; + if (code !== 'EPERM') return false; + try { + await fs.access(lockPath); + return true; + } catch { + return false; + } +} diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 3f68df80a..383f25dcc 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -563,11 +563,9 @@ const REGISTRY_LOCK_TIMEOUT_MS = 5_000; * registry-private lock namespace; the handle is kernel-owned on supported * platforms and crash-reclaimable by the existing fallback. * - * On timeout the transaction proceeds UNLOCKED rather than throwing: the lock - * closes a lost-update race that existed unguarded before #2716, so degrading - * to the old best-effort behaviour is strictly better than failing an - * `analyze`/`list`/`augment` outright on a wedged lock (a stale pid-reuse - * ghost on platforms without start-time verification can look live forever). + * On timeout the transaction fails closed: continuing unlocked would reintroduce + * the lost-update race this lock exists to prevent and can silently discard a + * concurrent registration. */ const withRegistryLock = async (operation: () => Promise): Promise => { let lock: IndexLockHandle | null = null; @@ -581,11 +579,13 @@ const withRegistryLock = async (operation: () => Promise): Promise => { logger.info('Waiting for another GitNexus process to finish a registry update…'), }); } catch (err) { - if (!(err instanceof IndexLockTimeoutError)) throw err; - logger.warn( - { timeoutMs: REGISTRY_LOCK_TIMEOUT_MS }, - 'Timed out waiting for the global registry lock; proceeding without it. A concurrent registry write may be lost.', - ); + if (err instanceof IndexLockTimeoutError) { + logger.error( + { timeoutMs: REGISTRY_LOCK_TIMEOUT_MS }, + 'Timed out waiting for the global registry lock; refusing an unlocked registry transaction.', + ); + } + throw err; } try { return await operation(); @@ -754,14 +754,11 @@ export const readRegistryStrict = async (): Promise => readRegi * Atomic tmp+rename: a crash mid-write can never leave a truncated * registry.json that the next load would treat as empty and silently drop * every registered repo (#2106 R9). The tmp path must stay per-write — the - * registry is the one file every gitnexus process on the machine writes, and - * `withRegistryLock` degrades to unlocked on timeout, so the write cannot rely - * on the lock to keep two writers off one staging path (#2888). - * - * `attempts` is forwarded to the rename retry; best-effort callers pass `1`. + * registry is the one file every gitnexus process on the machine writes (#2888). */ const writeRegistry = async (entries: RegistryEntry[], attempts?: number): Promise => { - await fs.mkdir(getGlobalDir(), { recursive: true }); + const dir = getGlobalDir(); + await fs.mkdir(dir, { recursive: true }); await writeFileAtomic( getGlobalRegistryPath(), JSON.stringify(sanitizeEntries(entries), null, 2), @@ -1569,8 +1566,6 @@ export const listRegisteredRepos = async (opts?: { try { await withRegistryLock(async () => { const fresh = await readRegistry(); - // attempts: 1 — the catch below discards a failure, so the rename - // backoff would only make every other process wait out this lock. await writeRegistry( fresh.filter((entry) => !pruned.has(entry.path)), 1, diff --git a/gitnexus/src/utils/process-identity.ts b/gitnexus/src/utils/process-identity.ts new file mode 100644 index 000000000..e7c99439c --- /dev/null +++ b/gitnexus/src/utils/process-identity.ts @@ -0,0 +1,40 @@ +import { execFileSync } from 'node:child_process'; + +export function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } +} + +export function readProcessStartTime(pid: number): string | undefined { + try { + const startedAt = + process.platform === 'win32' + ? execFileSync( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; if ($p) { $p.CreationDate.ToUniversalTime().ToString("O") }`, + ], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim() + : // `lstart` is rendered through localtime and the active locale, so the + // same live process yields a different string under a different TZ or + // LC_TIME. That string is a lock owner's identity, and a mismatch is + // read as PID reuse — an unpinned render lets one daemon reclaim a + // mutex another still holds. Pin both so the identity is absolute. + execFileSync('ps', ['-p', String(pid), '-o', 'lstart='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + env: { ...process.env, TZ: 'UTC', LC_ALL: 'C' }, + }).trim(); + return startedAt || undefined; + } catch { + return undefined; + } +} diff --git a/gitnexus/test/integration/watch-filesystem.test.ts b/gitnexus/test/integration/watch-filesystem.test.ts index 84fa4f8c3..51325e1ff 100644 --- a/gitnexus/test/integration/watch-filesystem.test.ts +++ b/gitnexus/test/integration/watch-filesystem.test.ts @@ -3,7 +3,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { startWatchFileLoop, type WatchFileLoop } from '../../src/cli/watch.js'; +import { startWatchFileLoop, type WatchFileLoop } from '../../src/cli/analyze-watch.js'; import { cleanupTempDir } from '../helpers/test-db.js'; const tempDirs: string[] = []; diff --git a/gitnexus/test/unit/auto-sync-analysis-worker.test.ts b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts new file mode 100644 index 000000000..b07a30308 --- /dev/null +++ b/gitnexus/test/unit/auto-sync-analysis-worker.test.ts @@ -0,0 +1,232 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, vi } from 'vitest'; + +const { autoHeapCapMbMock } = vi.hoisted(() => ({ autoHeapCapMbMock: vi.fn(() => 512) })); +vi.mock('../../src/core/ingestion/utils/effective-ram.js', () => ({ + autoHeapCapMb: autoHeapCapMbMock, +})); + +import { createAutoSyncAnalysisRunner } from '../../src/core/auto-sync/analysis-worker-launch.js'; + +function createChild() { + return Object.assign(new EventEmitter(), { + send: vi.fn(), + stdout: { resume: vi.fn() }, + stderr: { resume: vi.fn() }, + }); +} + +describe('auto-sync analysis worker', () => { + it('ignores progress and resolves from the terminal complete message', async () => { + const child = createChild(); + const forkWorker = vi.fn(() => child as any); + const run = createAutoSyncAnalysisRunner({ forkWorker }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + expect(forkWorker).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['--max-old-space-size=512']), + ); + child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).resolves.toEqual({ stats: { files: 3 } }); + expect(child.stdout.resume).toHaveBeenCalled(); + expect(child.stderr.resume).toHaveBeenCalled(); + }); + + it('rejects on a worker error even when no exit event follows', async () => { + const child = createChild(); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + const result = run('/tmp/repo', { branch: 'main' }, 50); + + child.emit('error', new Error('IPC disconnected')); + + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + await expect(result).rejects.toThrow('Auto-sync analyze worker error: IPC disconnected'); + }); + + it('rejects when the initial worker message cannot be sent', async () => { + const child = createChild(); + child.send.mockImplementationOnce(() => { + throw new Error('IPC channel closed'); + }); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + + await expect(result).rejects.toThrow( + 'Failed to start auto-sync analyze worker: IPC channel closed', + ); + expect(child.send).toHaveBeenNthCalledWith(2, { type: 'cancel' }); + }); + + it('preserves a worker terminal error', async () => { + const child = createChild(); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'progress', phase: 'parsing', percent: 20, message: 'Parsing' }); + child.emit('message', { type: 'error', message: 'parser crashed' }); + child.emit('exit', 1, null); + + await expect(result).rejects.toThrow('parser crashed'); + }); + + it('requests cancellation after timeout, reports it, and waits for exit', async () => { + const child = createChild(); + const timers: Array<() => void> = []; + const onCancellationRequested = vi.fn(); + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50, undefined, onCancellationRequested); + timers[0]!(); + + expect(onCancellationRequested).toHaveBeenCalledOnce(); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + let settled = false; + void result.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await Promise.resolve(); + expect(settled).toBe(false); + + child.emit('exit', 0, null); + await expect(result).rejects.toThrow('Analysis timed out after 50ms'); + }); + + it('keeps the timeout outcome when complete arrives after cancellation begins', async () => { + const child = createChild(); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + timers[0]!(); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).rejects.toThrow('Analysis timed out after 50ms'); + }); + + it('does not send cancellation after a terminal complete message', async () => { + const child = createChild(); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + child.emit('message', { type: 'complete', result: { stats: { files: 3 } } }); + child.emit('exit', 0, null); + + await expect(result).resolves.toEqual({ stats: { files: 3 } }); + expect(child.send).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(1); + }); + + it('divides the worker heap by the number of repos analyzed in parallel', async () => { + // Stubbed timers so neither run leaves a live timeout behind for the rest + // of the suite, and both promises are settled before the test returns. + const timers: Array<() => void> = []; + const forkChildren: ReturnType[] = []; + const forkWorker = vi.fn(() => { + const child = createChild(); + forkChildren.push(child); + return child as any; + }); + const run = createAutoSyncAnalysisRunner({ + forkWorker, + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const parallel = run('/tmp/repo', { branch: 'main' }, 50, undefined, undefined, 4); + expect(forkWorker).toHaveBeenLastCalledWith( + expect.any(String), + expect.arrayContaining(['--max-old-space-size=128']), + ); + + const solo = run('/tmp/repo', { branch: 'main' }, 50); + expect(forkWorker).toHaveBeenLastCalledWith( + expect.any(String), + expect.arrayContaining(['--max-old-space-size=512']), + ); + + for (const child of forkChildren) { + child.emit('message', { type: 'complete', result: { stats: { files: 1 } } }); + child.emit('exit', 0, null); + } + await expect(parallel).resolves.toEqual({ stats: { files: 1 } }); + await expect(solo).resolves.toEqual({ stats: { files: 1 } }); + }); + + it('stops waiting for a worker that never exits after cancellation', async () => { + const child = Object.assign(createChild(), { + unref: vi.fn(), + channel: { unref: vi.fn() }, + }); + const timers: Array<() => void> = []; + const run = createAutoSyncAnalysisRunner({ + forkWorker: vi.fn(() => child as any), + setTimeoutFn: vi.fn((callback: () => void) => { + timers.push(callback); + return timers.length as any; + }) as any, + clearTimeoutFn: vi.fn() as any, + }); + + const result = run('/tmp/repo', { branch: 'main' }, 50); + timers[0]!(); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + + // No 'exit' ever arrives — the worker is wedged past its safe point. + timers[1]!(); + + await expect(result).rejects.toThrow('did not exit within'); + // The parent stops waiting; the child is released, never killed. + expect(child.channel.unref).toHaveBeenCalled(); + expect(child.unref).toHaveBeenCalled(); + expect(child.send).toHaveBeenCalledTimes(2); + }); + + it('uses the same cancellation request for an aborted watch run', async () => { + const child = createChild(); + const controller = new AbortController(); + const run = createAutoSyncAnalysisRunner({ forkWorker: vi.fn(() => child as any) }); + + const result = run('/tmp/repo', { branch: 'main' }, 50, controller.signal); + controller.abort(); + expect(child.send).toHaveBeenLastCalledWith({ type: 'cancel' }); + + child.emit('exit', 0, null); + await expect(result).rejects.toThrow('Analysis cancelled'); + }); +}); diff --git a/gitnexus/test/unit/auto-sync-runner.test.ts b/gitnexus/test/unit/auto-sync-runner.test.ts new file mode 100644 index 000000000..2c261f0f0 --- /dev/null +++ b/gitnexus/test/unit/auto-sync-runner.test.ts @@ -0,0 +1,1949 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; + +import { + addRepoToGroup, + getAutoSyncRepoIdentity, + getConfiguredRepoPath, + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + resolveActualConcurrency, + runAutoSyncOnce, + startAutoSyncWatch, + stopAutoSyncWatch, +} from '../../src/core/auto-sync/index.js'; +import type { + AutoSyncConfig, + AutoSyncRunDeps, + AutoSyncWatchPaths, +} from '../../src/core/auto-sync/index.js'; + +const config: AutoSyncConfig = { + configPath: '/tmp/.gitnexus/watch_config.yml', + syncIntervalMinutes: 10, + repoGitTimeoutMs: 10_000, + analyzeTimeoutMs: 1_800_000, + maxConcurrency: 1, + analyzeFailureThreshold: 3, + projects: [ + { + localPath: '/tmp/repos', + groupName: 'back_end', + overwriteLocalChanges: false, + branches: ['master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }, + ], +}; + +const cloneRoot = { + root: '/tmp/repos', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + quarantineRetentionDays: 14, +}; +const verifiedWatchCommand = 'node /gitnexus/dist/cli/index.js auto-sync start'; +const verifiedProcessStartTime = 'Tue Aug 4 12:00:00 2026'; + +function withCloneRoot(deps: Partial): Partial { + return { + resolveCloneRoot: vi.fn(async () => cloneRoot), + ...deps, + }; +} + +async function writeWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, + ownerId = `owner-${pid}`, +): Promise { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.writeFile(paths.pidPath, `${pid}\n`); + await fs.writeFile( + paths.mutexPath, + `${JSON.stringify({ pid, ownerId: `mutex-${ownerId}`, processStartTime: verifiedProcessStartTime, hostname: os.hostname() })}\n`, + ); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid, ownerId, processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ + state: 'running', + pid, + ownerId, + updatedAt: '2026-06-30T00:00:00.000Z', + })}\n`, + ); + return ownerId; +} + +describe('auto-sync runner', () => { + it('runs clone, analyzes changed commits, registers the repo, and syncs changed groups', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/gitee.com/qts_server/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', + overwriteLocalChanges: false, + }, + ); + expect(deps.getCurrentBranch).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + 10_000, + ); + expect(deps.runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + { branch: 'master', skipAgentsMd: true, skipSkills: true }, + 1_800_000, + undefined, + undefined, + 1, + ); + expect(deps.registerRepo).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + expect.objectContaining({ + lastCommit: 'commit-2', + branch: 'master', + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + }), + { name: 'gitee.com/qts_server/qts_account' }, + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + status: 'success', + }), + ]); + }); + + it('registers into the branch slot the analyze worker placed the index in', async () => { + // Without this the parent always takes the primary/flat arm and relabels a + // pinned branch entry with whatever this tick happened to sync. + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(async () => 'master'), + getCurrentCommit: vi.fn(async () => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + resolveBranchPlacement: vi.fn(async () => ({ branch: 'master' })), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(deps.resolveBranchPlacement).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + 'master', + ); + expect(deps.registerRepo).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + expect.anything(), + { name: 'gitee.com/qts_server/qts_account', branch: 'master' }, + ); + }); + + it('syncs a group when a repo is newly added to the group', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('syncs a group after successful re-analysis even when membership already exists', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-3'), + runAnalysis: vi.fn(async () => ({ stats: { files: 2 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-2', + analyzedCommitId: 'commit-2', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(1); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('uses distinct registry and group identities for repositories with the same basename', async () => { + const duplicateConfig: AutoSyncConfig = { + ...config, + projects: [ + { + localPath: '/tmp/repos-a', + groupName: 'back_end', + branches: ['main'], + remoteUrls: ['git@github.com:team-a/service.git'], + }, + { + localPath: '/tmp/repos-b', + groupName: 'back_end', + branches: ['main'], + remoteUrls: ['git@gitlab.com:team-b/service.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + resolveCloneRoot: vi.fn(async (localPath: string) => ({ + ...cloneRoot, + root: localPath, + })), + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'main'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'service'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(duplicateConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.registerRepo).toHaveBeenNthCalledWith( + 1, + '/tmp/repos-a/github.com/team-a/service', + expect.anything(), + { name: 'github.com/team-a/service' }, + ); + expect(deps.registerRepo).toHaveBeenNthCalledWith( + 2, + '/tmp/repos-b/gitlab.com/team-b/service', + expect.anything(), + { name: 'gitlab.com/team-b/service' }, + ); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + duplicateConfig.projects[0], + 'github.com/team-a/service', + 'github.com/team-a/service', + ); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + duplicateConfig.projects[1], + 'gitlab.com/team-b/service', + 'gitlab.com/team-b/service', + ); + }); + + it('normalizes the .git suffix case in auto-sync repository identities', () => { + expect(getAutoSyncRepoIdentity('git@GitHub.com:team/service.GIT')).toBe( + 'github.com/team/service', + ); + }); + + it('skips analysis when commit id has not changed', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-1'), + runAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.analyzed).toBe(0); + expect(result.skippedAnalysis).toBe(1); + expect(deps.runAnalysis).not.toHaveBeenCalled(); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + }); + + it('retries a failed group sync on the next unchanged commit without re-analysis', async () => { + let persistedState: any = { + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + groupSyncPending: true, + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + }; + const syncGroupByName = vi + .fn() + .mockRejectedValueOnce(new Error('group temporarily unavailable')) + .mockResolvedValueOnce(undefined); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-1'), + runAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => structuredClone(persistedState)), + saveState: vi.fn(async (state) => { + persistedState = structuredClone(state); + }), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName, + getAvailableMemoryGB: vi.fn(() => 8), + }); + const runOptions = { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }; + + const first = await runAutoSyncOnce(config, runOptions); + const second = await runAutoSyncOnce(config, runOptions); + + expect(first.failed).toBe(1); + expect(second.failed).toBe(0); + expect(deps.runAnalysis).not.toHaveBeenCalled(); + expect(syncGroupByName).toHaveBeenCalledTimes(2); + expect( + persistedState['/tmp/repos/gitee.com/qts_server/qts_account|master'].groupSyncPending, + ).toBe(false); + }); + + it('uses remote identity under local_path as the clone target', async () => { + expect( + getConfiguredRepoPath( + config.projects[0], + 'qts_account', + 'git@gitee.com:qts_server/qts_account.git', + ), + ).toBe('/tmp/repos/gitee.com/qts_server/qts_account'); + + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(deps.cloneOrPull).toHaveBeenCalledWith( + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/gitee.com/qts_server/qts_account', + undefined, + { + allowedCloneRoot: '/tmp/repos', + expectedRepoName: 'qts_account', + quarantineRoot: '/tmp/.gitnexus/watch/quarantine', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'master', + overwriteLocalChanges: false, + }, + ); + }); + + it('passes watch cancellation controls to the isolated analysis runner', async () => { + const controller = new AbortController(); + const onAnalysisCancellationRequested = vi.fn(); + const runAnalysis = vi.fn(async () => ({ stats: { files: 1 } }) as any); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis, + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + signal: controller.signal, + onAnalysisCancellationRequested, + }); + + expect(runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + { branch: 'master', skipAgentsMd: true, skipSkills: true }, + 1_800_000, + controller.signal, + onAnalysisCancellationRequested, + 1, + ); + }); + + it('falls back through configured branches and analyzes the first pullable branch', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_remoteUrl, _targetDir, _progress, options) => { + if (options?.branch === 'missing') throw new Error('remote branch not found'); + return '/tmp/repos/gitee.com/qts_server/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 1, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/gitee.com/qts_server/qts_account', + undefined, + expect.objectContaining({ branch: 'missing' }), + ); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 2, + 'git@gitee.com:qts_server/qts_account.git', + '/tmp/repos/gitee.com/qts_server/qts_account', + undefined, + expect.objectContaining({ branch: 'develop' }), + ); + expect(deps.runAnalysis).toHaveBeenCalledWith( + '/tmp/repos/gitee.com/qts_server/qts_account', + { branch: 'develop', skipAgentsMd: true, skipSkills: true }, + 1_800_000, + undefined, + undefined, + 1, + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).not.toHaveBeenCalled(); + }); + + it('records branch_unavailable when all configured branches fail', async () => { + const warnLogger = vi.fn(); + const errorLogger = vi.fn(); + const branchConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], branches: ['missing', 'develop'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => { + throw new Error('remote branch not found'); + }), + getCurrentBranch: vi.fn(), + getCurrentCommit: vi.fn(), + runAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(branchConfig, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + branch: 'missing', + status: 'branch_unavailable', + }), + ]); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledTimes(2); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch missing unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch develop unavailable for git@gitee.com:qts_server/qts_account.git: remote branch not found', + ); + expect(errorLogger).toHaveBeenCalledTimes(1); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Repository sync failed for git@gitee.com:qts_server/qts_account.git; no configured branch could be pulled: missing: remote branch not found; develop: remote branch not found', + ); + }); + + it('records branch_unavailable when checkout ends on an unexpected branch', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'develop'), + getCurrentCommit: vi.fn(), + runAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is develop; trying next branch.', + ); + }); + + it('records branch_unavailable when the checked out repository is detached', async () => { + const warnLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => undefined), + getCurrentCommit: vi.fn(), + runAnalysis: vi.fn(), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: warnLogger, error: vi.fn() }, + }); + + expect(result).toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.getCurrentCommit).not.toHaveBeenCalled(); + expect(deps.runAnalysis).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).not.toHaveBeenCalled(); + expect(warnLogger).toHaveBeenCalledWith( + '[auto-sync] Branch master for git@gitee.com:qts_server/qts_account.git synced but current branch is ; trying next branch.', + ); + }); + + it('isolates repository and analysis failures without syncing groups for failed analysis', async () => { + const errorLogger = vi.fn(); + const failingConfig: AutoSyncConfig = { + ...config, + projects: [ + { + ...config.projects[0], + remoteUrls: [ + 'git@gitee.com:qts_server/failing_sync.git', + 'git@gitee.com:qts_server/qts_account.git', + ], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + if (remoteUrl.includes('failing_sync')) throw new Error('sync failed'); + return '/tmp/repos/gitee.com/qts_server/qts_account'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => { + throw new Error('analysis failed'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'success', + groupSyncPending: true, + lastSyncTime: '2026-06-29T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => true), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(failingConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 2 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(2); + expect(deps.registerRepo).not.toHaveBeenCalled(); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + failingConfig.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); + expect(deps.syncGroupByName).not.toHaveBeenCalled(); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + codeCommitId: 'commit-2', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining( + 'Repository sync failed for git@gitee.com:qts_server/failing_sync.git', + ), + ); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Analysis failed for /tmp/repos/gitee.com/qts_server/qts_account'), + ); + }); + + it('records the resolved target directory when a post-sync operation fails', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => { + throw new Error('git log failed'); + }), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).resolves.toEqual({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + remoteUrl: 'git@gitee.com:qts_server/qts_account.git', + localPath: '/tmp/repos/gitee.com/qts_server/qts_account', + status: 'sync_failed', + }), + ]); + }); + + it('isolates clone-root resolution failures to the affected project', async () => { + const isolatedConfig: AutoSyncConfig = { + ...config, + projects: [ + { ...config.projects[0], localPath: '/bad/repos' }, + { ...config.projects[0], localPath: '/tmp/repos' }, + ], + }; + const deps: Partial = withCloneRoot({ + resolveCloneRoot: vi.fn(async (localPath: string) => { + if (localPath === '/bad/repos') throw new Error('unsafe clone root'); + return cloneRoot; + }), + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(isolatedConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).resolves.toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); + expect(deps.cloneOrPull).toHaveBeenCalledTimes(1); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('persists state and commit info when repository registration fails', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => { + throw new Error('registry busy'); + }), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + lastAnalyzeStatus: 'failed', + analyzedCommitId: undefined, + lastAnalyzeError: 'Repository registration failed: registry busy', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Repository registration failed: registry busy', + ); + }); + + it('reports group sync failures after successful analysis', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => { + throw new Error('group sync failed'); + }), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 1 }); + expect(deps.addRepoToGroup).toHaveBeenCalledWith( + config.projects[0], + 'gitee.com/qts_server/qts_account', + 'gitee.com/qts_server/qts_account', + ); + expect(deps.syncGroupByName).toHaveBeenCalledWith('back_end'); + expect(errorLogger).toHaveBeenCalledWith( + expect.stringContaining('Group sync failed for back_end'), + ); + }); + + it('caps actual concurrency by available memory and runs clone/analyze work concurrently', async () => { + const events: string[] = []; + let releaseFirstClone: (() => void) | undefined; + const concurrentConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 4, + projects: [ + { + ...config.projects[0], + groupName: undefined, + remoteUrls: ['git@github.com:owner/one.git', 'git@gitlab.com:owner/two.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (remoteUrl) => { + events.push(`clone-start:${remoteUrl}`); + if (remoteUrl.includes('/one.git')) { + await new Promise((resolve) => { + releaseFirstClone = resolve; + setTimeout(resolve, 0); + }); + } else { + releaseFirstClone?.(); + } + events.push(`clone-end:${remoteUrl}`); + return remoteUrl.includes('/one.git') ? '/tmp/repos/one' : '/tmp/repos/two'; + }), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn((repoPath) => + repoPath.endsWith('/one') ? 'one-commit' : 'two-commit', + ), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 4), + }); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await runAutoSyncOnce(concurrentConfig, { deps, logger }); + + expect(result.synced).toBe(2); + expect(logger.info).toHaveBeenCalledWith( + '[auto-sync] Starting sync loop with max_concurrency=2 analyze_failure_threshold=3.', + ); + expect(events.slice(0, 2)).toEqual([ + 'clone-start:git@github.com:owner/one.git', + 'clone-start:git@gitlab.com:owner/two.git', + ]); + expect(deps.registerRepo).toHaveBeenCalledTimes(2); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('keeps same-basename remotes in distinct clone directories', async () => { + const duplicateConfig: AutoSyncConfig = { + ...config, + maxConcurrency: 2, + projects: [ + { + ...config.projects[0], + branches: ['main'], + remoteUrls: ['git@github.com:owner/repo.git', 'git@gitlab.com:group/repo.git'], + }, + ], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async (_url, targetDir) => targetDir), + getCurrentBranch: vi.fn(() => 'main'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async (_path, _meta, options) => options?.name ?? 'repo'), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(duplicateConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), + ).resolves.toEqual({ synced: 2, analyzed: 2, skippedAnalysis: 0, failed: 0 }); + + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 1, + 'git@github.com:owner/repo.git', + '/tmp/repos/github.com/owner/repo', + undefined, + expect.any(Object), + ); + expect(deps.cloneOrPull).toHaveBeenNthCalledWith( + 2, + 'git@gitlab.com:group/repo.git', + '/tmp/repos/gitlab.com/group/repo', + undefined, + expect.any(Object), + ); + expect(deps.saveState).toHaveBeenCalledTimes(1); + expect(deps.writeCommitInfo).toHaveBeenCalledTimes(1); + }); + + it('rejects non auto-sync SSH URLs at runner boundary', async () => { + const invalidConfig: AutoSyncConfig = { + ...config, + projects: [{ ...config.projects[0], remoteUrls: ['https://github.com/owner/repo.git'] }], + }; + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(invalidConfig, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + expect(result.failed).toBe(1); + expect(deps.cloneOrPull).not.toHaveBeenCalled(); + }); + + it('resets consecutive analyze failures when the code commit changes, then records this failure', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => { + throw new Error('parser crashed\nwith stack'); + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + // New code commit (commit-1 → commit-2) zeros this before the failed analysis increments to 1. + analyzeConsecutiveFailures: 1, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 1, + lastAnalyzeError: 'parser crashed with stack', + lastAnalyzeStatus: 'failed', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'failed', + analyzeConsecutiveFailures: 1, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed with stack', + }), + ]); + expect(errorLogger).toHaveBeenCalledWith( + '[auto-sync] Analysis failed for /tmp/repos/gitee.com/qts_server/qts_account; consecutive failures 1/3: parser crashed with stack', + ); + }); + + it('records a null analysis failure without masking it with a TypeError', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => { + throw null; + }), + registerRepo: vi.fn(), + loadState: vi.fn(async () => ({})), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + await expect( + runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }), + ).resolves.toEqual({ synced: 1, analyzed: 0, skippedAnalysis: 0, failed: 1 }); + + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + lastAnalyzeError: 'null', + }), + }), + ); + }); + + it('retries analysis on a new commit after consecutive failures reached the threshold', async () => { + const errorLogger = vi.fn(); + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: errorLogger }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result).toEqual({ synced: 1, analyzed: 1, skippedAnalysis: 0, failed: 0 }); + expect(deps.runAnalysis).toHaveBeenCalledTimes(1); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 0, + lastAnalyzeError: undefined, + lastAnalyzeStatus: 'success', + }), + }), + ); + expect(deps.writeCommitInfo).toHaveBeenCalledWith([ + expect.objectContaining({ + status: 'success', + analyzeConsecutiveFailures: 0, + analyzeFailureThreshold: 3, + lastAnalyzeError: undefined, + }), + ]); + expect(errorLogger).not.toHaveBeenCalled(); + }); + + it('clears prior analyze failure count after a successful analyze', async () => { + const deps: Partial = withCloneRoot({ + cloneOrPull: vi.fn(async () => '/tmp/repos/gitee.com/qts_server/qts_account'), + getCurrentBranch: vi.fn(() => 'master'), + getCurrentCommit: vi.fn(() => 'commit-2'), + runAnalysis: vi.fn(async () => ({ stats: { files: 1 } }) as any), + registerRepo: vi.fn(async () => 'qts_account'), + loadState: vi.fn(async () => ({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': { + codeCommitId: 'commit-1', + analyzedCommitId: 'commit-1', + lastAnalyzeStatus: 'failed', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-01-01T00:00:00.000Z', + }, + })), + saveState: vi.fn(async () => {}), + writeCommitInfo: vi.fn(async () => {}), + addRepoToGroup: vi.fn(async () => false), + syncGroupByName: vi.fn(async () => {}), + getAvailableMemoryGB: vi.fn(() => 8), + }); + + const result = await runAutoSyncOnce(config, { + deps, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + now: () => new Date('2026-06-30T00:00:00.000Z'), + }); + + expect(result.analyzed).toBe(1); + expect(deps.saveState).toHaveBeenCalledWith( + expect.objectContaining({ + '/tmp/repos/gitee.com/qts_server/qts_account|master': expect.objectContaining({ + analyzeConsecutiveFailures: 0, + lastAnalyzeError: undefined, + lastAnalyzeStatus: 'success', + }), + }), + ); + }); + + it('resolves actual concurrency from configured value and memory', () => { + expect(resolveActualConcurrency(8, 10)).toBe(5); + expect(resolveActualConcurrency(8, 1)).toBe(1); + expect(resolveActualConcurrency(2, 10)).toBe(2); + }); + + it('detects existing groupPath to registryName mappings as already joined', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-group-')); + try { + process.env.GITNEXUS_HOME = tempDir; + const groupDir = path.join(tempDir, 'groups', 'back_end'); + await fs.mkdir(groupDir, { recursive: true }); + await fs.writeFile( + path.join(groupDir, 'group.yaml'), + ['version: 1', 'name: back_end', 'repos:', ' hr/hiring/backend: qts_account'].join('\n'), + ); + + await expect( + addRepoToGroup({ groupName: 'back_end' }, 'hr/hiring/backend', 'qts_account'), + ).resolves.toBe(false); + + await expect(fs.readFile(path.join(groupDir, 'group.yaml'), 'utf-8')).resolves.toContain( + 'hr/hiring/backend: qts_account', + ); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); + +describe('auto-sync starter', () => { + it('registers a clearable timer with a valid fixed config', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + const setIntervalFn = vi.fn(() => timer) as unknown as typeof setInterval; + const clearIntervalFn = vi.fn() as unknown as typeof clearInterval; + const runOnce = vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })); + const stderr = { write: vi.fn() }; + + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' group_name: back_end', + ' branch: master', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + setIntervalFn, + clearIntervalFn, + runOnce, + stderr, + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + + expect(handle).not.toBeNull(); + expect(runOnce).toHaveBeenCalledTimes(1); + expect(setIntervalFn).toHaveBeenCalledWith(expect.any(Function), 300_000); + expect(timer.unref).toHaveBeenCalled(); + await vi.waitFor(() => { + expect(stderr.write).toHaveBeenCalledWith( + expect.stringContaining('[auto-sync] Watch loop started at '), + ); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch loop finished: synced=0 analyzed=0 skipped=0 failed=0.\n', + ); + }); + + await handle?.stop(); + + expect(clearIntervalFn).toHaveBeenCalledWith(timer); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('skips overlapping scheduled runs while a previous run is active', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + let scheduled: (() => void) | undefined; + const setIntervalFn = vi.fn((fn: () => void) => { + scheduled = fn; + return timer; + }) as unknown as typeof setInterval; + const stderr = { write: vi.fn() }; + const releaseRuns: Array<() => void> = []; + const runOnce = vi.fn( + () => + new Promise((resolve) => { + releaseRuns.push(() => + resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }), + ); + }), + ); + let handle: Awaited> | undefined; + + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + handle = await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(1); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Previous run is still active; skipping overlapping run.\n', + ); + + releaseRuns.shift()?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(2); + releaseRuns.shift()?.(); + await handle?.stop(); + handle = undefined; + } finally { + releaseRuns.splice(0).forEach((release) => release()); + await handle?.stop(); + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not start a new run from a queued interval tick after stop', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const timer = { unref: vi.fn() }; + let scheduled: (() => void) | undefined; + const setIntervalFn = vi.fn((fn: () => void) => { + scheduled = fn; + return timer; + }) as unknown as typeof setInterval; + const stderr = { write: vi.fn() }; + let releaseRun: (() => void) | undefined; + const runOnce = vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }), + ); + let handle: Awaited> | undefined; + + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + handle = await startAutoSyncWatch({ setIntervalFn, runOnce, stderr }); + expect(runOnce).toHaveBeenCalledTimes(1); + + const stopping = handle!.stop(); + releaseRun?.(); + await stopping; + handle = undefined; + scheduled?.(); + + expect(runOnce).toHaveBeenCalledTimes(1); + } finally { + releaseRun?.(); + await handle?.stop(); + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('cancels the active run before removing watch ownership files', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-starter-')); + const cancelled = vi.fn(); + const runOnce = vi.fn( + (_config, options) => + new Promise((resolve) => { + options?.signal?.addEventListener( + 'abort', + () => { + cancelled(); + resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }, + { once: true }, + ); + }), + ); + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + const handle = await startAutoSyncWatch({ + runOnce, + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + const paths = getAutoSyncWatchPaths(tempDir); + await handle!.stop(); + + expect(cancelled).toHaveBeenCalledTimes(1); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses a second running watch for the same GITNEXUS_HOME', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + await writeWatchOwner(paths, 12345); + const handle = await startAutoSyncWatch({ + paths, + stderr, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch is already running with pid 12345.\n', + ); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('recovers an abandoned watch mutex after the owner exits', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + process.env.GITNEXUS_HOME = tempDir; + await writeWatchOwner(paths, 12345, 'abandoned-owner'); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + stderr, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + keepAlive: false, + deps: { isProcessAlive: vi.fn(() => false) }, + }); + + expect(handle).not.toBeNull(); + expect(await fs.readFile(paths.pidPath, 'utf-8')).toBe(`${process.pid}\n`); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).not.toContain('abandoned-owner'); + await handle?.stop(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a half-initialized lease when pid has not been written yet', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const handle = await startAutoSyncWatch({ + paths, + stderr, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }, + }); + + expect(handle).toBeNull(); + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch is already running with pid 12345.\n', + ); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a live half-initialized lease when stop runs before pid is written', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const stderr = { write: vi.fn() }; + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'starting-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr, + deps: { isProcessAlive: vi.fn(() => true) }, + }), + ).resolves.toBe('refused'); + + expect(stderr.write).toHaveBeenCalledWith( + '[auto-sync] Watch appears to be starting with pid 12345; pid file is not ready.\n', + ); + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('starting-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.statusPath)).rejects.toThrow(); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not delete a stale half-initialized lease from the stopper', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + await fs.mkdir(paths.mutexPath); + await fs.writeFile( + paths.ownerPath, + `${JSON.stringify({ pid: 12345, ownerId: 'stale-owner', processStartTime: verifiedProcessStartTime, createdAt: '2026-06-30T00:00:00.000Z' })}\n`, + ); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { isProcessAlive: vi.fn(() => false) }, + }), + ).resolves.toBe('refused'); + + expect(await fs.readFile(paths.ownerPath, 'utf-8')).toContain('stale-owner'); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.statusPath)).rejects.toThrow(); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports not_running when no watch lease exists', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + try { + await expect( + stopAutoSyncWatch({ + paths: getAutoSyncWatchPaths(tempDir), + stderr: { write: vi.fn() }, + }), + ).resolves.toBe('not_running'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('writes an owner-fenced stop request without deleting a live watch lease', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + const ownerId = await writeWatchOwner(paths, 12345, 'verified-owner'); + + await expect( + stopAutoSyncWatch({ + paths, + timeoutMs: 0, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }, + }), + ).resolves.toBe('timeout'); + + expect( + JSON.parse( + await fs.readFile( + path.join(path.dirname(paths.pidPath), `watch.stop.${ownerId}.json`), + 'utf-8', + ), + ), + ).toMatchObject({ + pid: 12345, + ownerId, + processStartTime: verifiedProcessStartTime, + requestedAt: expect.any(String), + }); + await expect(fs.readFile(paths.pidPath, 'utf-8')).resolves.toBe('12345\n'); + await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('refuses to request stop for a reused pid', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await writeWatchOwner(paths, 12345); + + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }, + }), + ).resolves.toBe('refused'); + + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }), + ).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('not a GitNexus auto-sync process'), + updatedAt: '2026-06-30T00:00:00.000Z', + }); + await expect(fs.readdir(path.dirname(paths.pidPath))).resolves.not.toContainEqual( + expect.stringMatching(/^watch\.stop\./), + ); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('ignores a tampered ownerId that would escape the watch directory', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await writeWatchOwner(paths, 12345, '../../victim'); + await expect( + stopAutoSyncWatch({ + paths, + stderr: { write: vi.fn() }, + deps: { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }, + }), + ).resolves.toBe('refused'); + await expect(fs.readdir(path.dirname(paths.pidPath))).resolves.not.toContainEqual( + expect.stringMatching(/victim/), + ); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('does not trust a stored error status for an unverified live pid', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + const ownerId = await writeWatchOwner(paths, 12345); + await fs.writeFile( + paths.statusPath, + `${JSON.stringify({ + state: 'error', + pid: 12345, + ownerId, + message: 'stale stored failure', + updatedAt: '2026-06-30T00:00:00.000Z', + })}\n`, + ); + + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => 'node unrelated-service.js'), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }), + ).resolves.toMatchObject({ + state: 'error', + pid: 12345, + message: expect.stringContaining('not a GitNexus auto-sync process'), + updatedAt: '2026-06-30T00:00:00.000Z', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('preserves stored updatedAt when the watch pid is stale', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + try { + await writeWatchOwner(paths, 12345); + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => false), + }), + ).resolves.toMatchObject({ + state: 'stale', + pid: 12345, + updatedAt: '2026-06-30T00:00:00.000Z', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('stops a watch only when its own owner-fenced request is polled', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + const callbacks: Array<() => void> = []; + const timer = { unref: vi.fn() }; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + const handle = await startAutoSyncWatch({ + paths, + keepAlive: false, + setIntervalFn: vi.fn((callback: () => void) => { + callbacks.push(callback); + return timer; + }) as unknown as typeof setInterval, + clearIntervalFn: vi.fn() as unknown as typeof clearInterval, + runOnce: vi.fn(async () => ({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 })), + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + expect(handle).not.toBeNull(); + const owner = JSON.parse(await fs.readFile(paths.ownerPath, 'utf-8')); + await fs.writeFile( + path.join(path.dirname(paths.pidPath), `watch.stop.${owner.ownerId}.json`), + `${JSON.stringify({ + pid: process.pid, + ownerId: owner.ownerId, + processStartTime: verifiedProcessStartTime, + requestedAt: new Date().toISOString(), + })}\n`, + ); + + callbacks[0]!(); + await vi.waitFor(async () => expect(fs.access(paths.pidPath)).rejects.toThrow()); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('reports cancelling until a timed-out analysis run settles', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + let releaseRun!: () => void; + let requestCancellation!: () => void; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + const handle = await startAutoSyncWatch({ + paths, + keepAlive: false, + runOnce: vi.fn( + (_config, options) => + new Promise((resolve) => { + requestCancellation = options.onAnalysisCancellationRequested; + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }), + ), + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + requestCancellation(); + await vi.waitFor(async () => { + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }), + ).resolves.toMatchObject({ state: 'cancelling' }); + }); + + releaseRun(); + await vi.waitFor(async () => { + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }), + ).resolves.toMatchObject({ state: 'running' }); + }); + await handle?.stop(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it('keeps watch ownership while stop waits for an active run to settle', async () => { + const previousHome = process.env.GITNEXUS_HOME; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-auto-sync-watch-')); + const paths = getAutoSyncWatchPaths(tempDir); + let releaseRun!: () => void; + try { + process.env.GITNEXUS_HOME = tempDir; + await fs.writeFile( + path.join(tempDir, 'watch_config.yml'), + [ + 'sync_interval_minutes: 5', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + const handle = await startAutoSyncWatch({ + paths, + keepAlive: false, + runOnce: vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve({ synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }); + }), + ), + deps: { readProcessStartTime: vi.fn(() => verifiedProcessStartTime) }, + }); + const stopping = handle!.stop(); + + await vi.waitFor(async () => { + await expect( + readAutoSyncWatchStatus(paths, { + isProcessAlive: vi.fn(() => true), + readProcessCommand: vi.fn(() => verifiedWatchCommand), + readProcessStartTime: vi.fn(() => verifiedProcessStartTime), + }), + ).resolves.toMatchObject({ state: 'stopping' }); + }); + await expect(fs.access(paths.pidPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.ownerPath)).resolves.toBeUndefined(); + await expect(fs.access(paths.mutexPath)).resolves.toBeUndefined(); + + releaseRun(); + await stopping; + await expect(fs.access(paths.pidPath)).rejects.toThrow(); + await expect(fs.access(paths.ownerPath)).rejects.toThrow(); + await expect(fs.access(paths.mutexPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = previousHome; + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/auto-sync.test.ts b/gitnexus/test/unit/auto-sync.test.ts new file mode 100644 index 000000000..1b0110023 --- /dev/null +++ b/gitnexus/test/unit/auto-sync.test.ts @@ -0,0 +1,730 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + extractRepoNameFromRemoteUrl, + getAutoSyncMutexPath, + getAutoSyncStatePath, + getAutoSyncWatchDir, + getProjectCommitInfoPath, + loadAutoSyncConfig, + parseAutoSyncConfig, + parseBranchCandidates, + parseDurationMs, + quarantineAutoSyncPartial, + resolveConfiguredCloneRoot, + loadAutoSyncState, + resetAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + validateAutoSyncRemoteUrl, + validateAutoSyncBranchName, + writeProjectCommitInfo, +} from '../../src/core/auto-sync/index.js'; +import { acquireFileLock } from '../../src/storage/file-lock.js'; + +describe('auto-sync', () => { + let tempDir: string; + let gitnexusHome: string; + let oldHome: string | undefined; + + beforeEach(async () => { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + tempDir = await fs.realpath(await fs.mkdtemp(path.join(base, 'gitnexus-auto-sync-'))); + gitnexusHome = path.join(tempDir, '.gitnexus'); + await fs.mkdir(gitnexusHome); + oldHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = gitnexusHome; + }); + + afterEach(async () => { + if (oldHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = oldHome; + await fs.rm(tempDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('places watch runtime artifacts under the watch directory by default', () => { + expect(getAutoSyncWatchDir(gitnexusHome)).toBe(path.join(gitnexusHome, 'watch')); + expect(getAutoSyncMutexPath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'watch.mutex'), + ); + expect(getAutoSyncStatePath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'auto-sync-state.json'), + ); + expect(getProjectCommitInfoPath(gitnexusHome)).toBe( + path.join(gitnexusHome, 'watch', 'project_commit_info.txt'), + ); + }); + + it('refuses to reset state while the watch mutex is held', async () => { + const statePath = getAutoSyncStatePath(gitnexusHome); + const infoPath = getProjectCommitInfoPath(gitnexusHome); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, '{"kept":true}\n'); + await fs.writeFile(infoPath, 'kept\n'); + const release = await acquireFileLock(getAutoSyncMutexPath(gitnexusHome)); + + try { + await expect(resetAutoSyncState(gitnexusHome)).resolves.toBe(false); + await expect(fs.readFile(statePath, 'utf-8')).resolves.toContain('kept'); + await expect(fs.readFile(infoPath, 'utf-8')).resolves.toBe('kept\n'); + } finally { + await release(); + } + }); + + it('resets derived state while holding the watch mutex', async () => { + const statePath = getAutoSyncStatePath(gitnexusHome); + const infoPath = getProjectCommitInfoPath(gitnexusHome); + const mutexPath = getAutoSyncMutexPath(gitnexusHome); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, '{}\n'); + await fs.writeFile(infoPath, 'derived\n'); + + await expect(resetAutoSyncState(gitnexusHome)).resolves.toBe(true); + + await expect(fs.access(statePath)).rejects.toThrow(); + await expect(fs.access(infoPath)).rejects.toThrow(); + await expect(fs.access(mutexPath)).rejects.toThrow(); + }); + + it('loads watch_config.yml from GITNEXUS_HOME and normalizes branch candidates', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 120', + 'max_concurrency: 3', + 'repo_git_timeout: 12s', + 'analyze_timeout: 45m', + 'analyze_failure_threshold: 2', + 'projects:', + ' - local_path: /tmp/repos', + ' group_name: back_end', + ' overwrite_local_changes: true', + ' branches: [test, master, test]', + ' remote_urls:', + ' - git@gitee.com:qts_server/qts_account.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config to load'); + expect(loaded.config.configPath).toBe(path.join(gitnexusHome, 'watch_config.yml')); + expect(loaded.config.syncIntervalMinutes).toBe(120); + expect(loaded.config.maxConcurrency).toBe(3); + expect(loaded.config.repoGitTimeoutMs).toBe(12_000); + expect(loaded.config.analyzeTimeoutMs).toBe(2_700_000); + expect(loaded.config.analyzeFailureThreshold).toBe(2); + expect(loaded.config.projects[0]).toMatchObject({ + localPath: '/tmp/repos', + groupName: 'back_end', + overwriteLocalChanges: true, + branches: ['test', 'master'], + remoteUrls: ['git@gitee.com:qts_server/qts_account.git'], + }); + }); + + it('defaults repo_git_timeout and max_concurrency and allows empty group_name', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' group_name: ""', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(true); + if (!loaded.ok) throw new Error('expected config'); + expect(loaded.config.repoGitTimeoutMs).toBe(10_000); + expect(loaded.config.analyzeTimeoutMs).toBe(300_000); + expect(loaded.config.maxConcurrency).toBe(1); + expect(loaded.config.analyzeFailureThreshold).toBe(3); + expect(loaded.config.projects[0].groupName).toBeUndefined(); + expect(loaded.config.projects[0].overwriteLocalChanges).toBe(false); + }); + + it('rejects boolean max_concurrency instead of coercing it to 1', () => { + expect(() => + parseAutoSyncConfig( + [ + 'sync_interval_minutes: 10', + 'max_concurrency: true', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + '/tmp/watch_config.yml', + ), + ).toThrow('max_concurrency must be a positive integer'); + }); + + it('rejects repo_git_timeout values above the Node timer limit', () => { + expect(() => + parseAutoSyncConfig( + [ + 'sync_interval_minutes: 10', + 'repo_git_timeout: 2147483648ms', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: main', + ' remote_urls:', + ' - https://github.com/owner/repo.git', + ].join('\n'), + '/tmp/watch_config.yml', + ), + ).toThrow('repo_git_timeout must not exceed 2147483647ms'); + }); + + it('rejects a repo_git_timeout that exceeds the interval or an hour', () => { + const config = (timeout: string) => + [ + 'sync_interval_minutes: 10', + `repo_git_timeout: ${timeout}`, + 'projects:', + ' - local_path: /tmp/repos', + ' branch: main', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'); + + // A bare number means seconds, so this is ~7 days, not 10 minutes. + expect(() => parseAutoSyncConfig(config('600000'), '/tmp/watch_config.yml')).toThrow( + 'a bare number is interpreted as seconds', + ); + expect(() => parseAutoSyncConfig(config('600000ms'), '/tmp/watch_config.yml')).not.toThrow(); + }); + + it('rejects analyze_timeout values above half the sync interval', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'analyze_timeout: 6m', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain( + 'analyze_timeout must not exceed half of sync_interval_minutes (5m)', + ); + }); + + it('rejects invalid analyze_failure_threshold values', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'analyze_failure_threshold: 1', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('analyze_failure_threshold must be an integer >= 2'); + }); + + it('reports missing config without throwing', async () => { + const loaded = await loadAutoSyncConfig(); + + expect(loaded).toEqual({ + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${path.join(gitnexusHome, 'watch_config.yml')}. Auto sync is skipped.`, + }); + }); + + it('reports invalid config without throwing', async () => { + await fs.writeFile(path.join(gitnexusHome, 'watch_config.yml'), 'projects: []\n'); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.reason).toBe('invalid'); + expect(loaded.message).toContain('[auto-sync] Invalid watch_config.yml:'); + expect(loaded.message).toContain('sync_interval_minutes must be a positive integer'); + expect(loaded.message).toContain('projects must contain at least one project'); + }); + + it('rejects missing, relative, and traversal local_path values at config load', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: ../repos', + ' branch: master', + ' remote_urls:', + ' - git@github.com:team/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('local_path must be an absolute path'); + }); + + it('hard-fails unsafe configured clone roots', async () => { + await expect(resolveConfiguredCloneRoot('/')).rejects.toThrow('unsafe auto-sync clone root'); + await expect(resolveConfiguredCloneRoot(os.homedir())).rejects.toThrow( + 'unsafe auto-sync clone root', + ); + await expect( + resolveConfiguredCloneRoot(path.join(await fs.realpath(os.tmpdir()), 'repos')), + ).rejects.toThrow('unsafe auto-sync clone root'); + const root = path.join(tempDir, 'repos'); + await expect(resolveConfiguredCloneRoot(`${root}/../repos`)).rejects.toThrow('normalized'); + }); + + it('rejects GitNexus internal directory descendants as clone roots', async () => { + for (const internalDir of ['groups', 'indexes', 'quarantine']) { + const root = path.join(gitnexusHome, internalDir, 'repo-root'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('GitNexus internal directory'); + } + }); + + it('allows the default GitNexus repos directory as an auto-sync clone root', async () => { + const root = path.join(gitnexusHome, 'repos'); + await fs.mkdir(root, { recursive: true }); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects symlinks in configured clone root paths', + async () => { + const realRoot = path.join(tempDir, 'real-root'); + const linkRoot = path.join(tempDir, 'link-root'); + await fs.mkdir(realRoot); + await fs.symlink(realRoot, linkRoot); + + await expect(resolveConfiguredCloneRoot(linkRoot)).rejects.toThrow('symlink'); + }, + ); + + it('resolves safe configured clone roots and reports quarantine retention', async () => { + const root = path.join(tempDir, 'repos'); + await fs.mkdir(root); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + quarantineRetentionDays: 14, + }), + ); + }); + + it('removes expired quarantine entries while preserving recent and unrelated files', async () => { + const root = path.join(tempDir, 'repos'); + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + const expired = path.join(quarantineRoot, 'auto-sync-expired-repo'); + const recent = path.join(quarantineRoot, 'auto-sync-recent-repo'); + const unrelated = path.join(quarantineRoot, 'operator-note.txt'); + await fs.mkdir(expired, { recursive: true }); + await fs.mkdir(recent); + await fs.writeFile(unrelated, 'keep'); + const old = new Date(Date.now() - 15 * 24 * 60 * 60 * 1_000); + await fs.utimes(expired, old, old); + + await resolveConfiguredCloneRoot(root); + + await expect(fs.access(expired)).rejects.toThrow(); + await expect(fs.access(recent)).resolves.toBeUndefined(); + await expect(fs.readFile(unrelated, 'utf-8')).resolves.toBe('keep'); + }); + + it('keeps only the newest quarantine entries per repository', async () => { + const root = path.join(tempDir, 'repos'); + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + await fs.mkdir(quarantineRoot, { recursive: true }); + const uuid = '00000000-0000-4000-8000-000000000000'; + const entryName = (stamp: string, repo: string) => `auto-sync-${stamp}-4242-${uuid}-${repo}`; + // Seven ticks of the same failing repo; age alone would keep them all. + const busy = ['01', '02', '03', '04', '05', '06', '07'].map((n) => + entryName(`2026-08-2${n}T00-00-00-000Z`, 'busy-repo'), + ); + const quiet = ['01', '02'].map((n) => entryName(`2026-08-2${n}T00-00-00-000Z`, 'quiet-repo')); + for (const name of [...busy, ...quiet]) { + await fs.mkdir(path.join(quarantineRoot, name), { recursive: true }); + await fs.writeFile(path.join(quarantineRoot, `${name}.README.txt`), 'note'); + } + + await resolveConfiguredCloneRoot(root); + + const survivors = await fs.readdir(quarantineRoot); + for (const name of busy.slice(-5)) { + expect(survivors).toContain(name); + expect(survivors).toContain(`${name}.README.txt`); + } + for (const name of busy.slice(0, 2)) { + expect(survivors).not.toContain(name); + expect(survivors).not.toContain(`${name}.README.txt`); + } + // A repo below the cap is untouched. + for (const name of quiet) expect(survivors).toContain(name); + }); + + it('falls back to copy and remove when quarantine crosses filesystems', async () => { + const target = path.join(tempDir, 'partial-repo'); + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'partial.txt'), 'partial'); + vi.spyOn(fs, 'rename').mockRejectedValueOnce( + Object.assign(new Error('cross-device link'), { code: 'EXDEV' }), + ); + + const destination = await quarantineAutoSyncPartial(target, quarantineRoot); + + await expect(fs.readFile(path.join(destination, 'partial.txt'), 'utf-8')).resolves.toBe( + 'partial', + ); + await expect(fs.access(target)).rejects.toThrow(); + }); + + it('gives concurrent partial clone quarantines unique destinations', async () => { + const quarantineRoot = path.join(gitnexusHome, 'watch', 'quarantine'); + const first = path.join(tempDir, 'one', 'partial-repo'); + const second = path.join(tempDir, 'two', 'partial-repo'); + await Promise.all([ + fs.mkdir(first, { recursive: true }), + fs.mkdir(second, { recursive: true }), + ]); + + const [firstDestination, secondDestination] = await Promise.all([ + quarantineAutoSyncPartial(first, quarantineRoot), + quarantineAutoSyncPartial(second, quarantineRoot), + ]); + + expect(firstDestination).not.toBe(secondDestination); + await expect(fs.access(firstDestination)).resolves.toBeUndefined(); + await expect(fs.access(secondDestination)).resolves.toBeUndefined(); + await expect(fs.access(first)).rejects.toThrow(); + await expect(fs.access(second)).rejects.toThrow(); + }); + + it('rejects group-writable configured clone roots', async () => { + if (process.platform === 'win32') return; + const root = path.join(tempDir, 'group-writable-repos'); + await fs.mkdir(root, { mode: 0o770 }); + await fs.chmod(root, 0o770); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('group-writable'); + }); + + it('rejects sticky world-writable configured clone roots', async () => { + if (process.platform === 'win32') return; + const root = path.join(tempDir, 'sticky-world-writable-repos'); + await fs.mkdir(root); + await fs.chmod(root, 0o1777); + + await expect(resolveConfiguredCloneRoot(root)).rejects.toThrow('world-writable'); + }); + + it('creates missing configured clone roots before watch clone work', async () => { + const root = path.join(tempDir, 'missing-repos'); + + await expect(resolveConfiguredCloneRoot(root)).resolves.toEqual( + expect.objectContaining({ + root, + quarantineRoot: path.join(gitnexusHome, 'watch', 'quarantine'), + }), + ); + expect((await fs.stat(root)).isDirectory()).toBe(true); + }); + + it('parses branch strings and arrays with trimming and de-duplication', () => { + expect(parseBranchCandidates('test, master, test')).toEqual(['test', 'master']); + expect(parseBranchCandidates(['develop,master', 'develop'])).toEqual(['develop', 'master']); + }); + + it('rejects unsafe auto-sync branch names', () => { + expect(() => validateAutoSyncBranchName('feature/good-branch')).not.toThrow(); + expect(() => validateAutoSyncBranchName('foo./bar')).toThrow('trailing-dot'); + expect(() => validateAutoSyncBranchName('/main')).toThrow('must not start'); + expect(() => validateAutoSyncBranchName('-upload-pack=evil')).toThrow('must not start'); + expect(() => validateAutoSyncBranchName('feature bad')).toThrow('whitespace'); + expect(() => validateAutoSyncBranchName('feature..bad')).toThrow('must not contain ".."'); + expect(() => validateAutoSyncBranchName('bad:ref')).toThrow('not allowed'); + expect(() => validateAutoSyncBranchName('feature.')).toThrow('must not end'); + expect(() => validateAutoSyncBranchName('feature/')).toThrow('must not end'); + expect(() => validateAutoSyncBranchName('feature//branch')).toThrow('consecutive'); + expect(() => validateAutoSyncBranchName('feature@{x')).toThrow('must not contain "@{"'); + expect(() => validateAutoSyncBranchName('.hidden')).toThrow('hidden'); + expect(() => validateAutoSyncBranchName('foo/bar.lock')).toThrow('.lock'); + }); + + it('extracts safe repository names from remote URLs', () => { + expect(extractRepoNameFromRemoteUrl('git@gitee.com:qts_server/qts_account.git')).toBe( + 'qts_account', + ); + expect(extractRepoNameFromRemoteUrl('git@gitlab.com:team/subgroup/repo-name.git')).toBe( + 'repo-name', + ); + }); + + it('rejects unsafe repository names without sanitizing them', () => { + // Rejected by the URL validator now, so the operator learns at config load + // rather than once per tick from inside the sync loop. + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/repo$name.git')).toThrow( + 'repository name must use only', + ); + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/repo\\name.git')).toThrow( + 'repository name must use only', + ); + expect(() => extractRepoNameFromRemoteUrl('git@github.com:team/..')).toThrow('traversal'); + }); + + it('allows only github, gitlab, and gitee SSH SCP remote URLs', () => { + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@github.com:im-fan/multica.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@gitlab.com:group/subgroup/repo.git')).not.toThrow(); + expect(() => + validateAutoSyncRemoteUrl('git@gitee.com:qts-ops/qts-code-engineering.git'), + ).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('https://github.com/owner/repo.git')).toThrow( + 'must use', + ); + expect(() => validateAutoSyncRemoteUrl('ssh://git@github.com/owner/repo.git')).toThrow( + 'must use', + ); + expect(() => validateAutoSyncRemoteUrl('user@github.com:owner/repo.git')).toThrow('must use'); + expect(() => validateAutoSyncRemoteUrl('git@example.com:owner/repo.git')).toThrow( + 'host must be', + ); + // Traversal is a whole segment; consecutive dots inside a name are not. + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/foo..bar.git')).not.toThrow(); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/../escape.git')).toThrow( + 'traversal', + ); + // A separator smuggled into a segment is traversal on Windows even though + // the segment is not literally `..`. + expect(() => validateAutoSyncRemoteUrl(String.raw`git@github.com:..\..\outside/repo`)).toThrow( + 'traversal', + ); + expect(() => validateAutoSyncRemoteUrl(String.raw`git@github.com:owner\..\..\x/repo`)).toThrow( + 'traversal', + ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo.git?ref=main')).toThrow( + 'must not include query strings or fragments', + ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/repo.git#main')).toThrow( + 'must not include query strings or fragments', + ); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner/')).toThrow('path must include'); + expect(() => validateAutoSyncRemoteUrl('git@github.com:owner//repo')).toThrow( + 'path must include', + ); + }); + + it('parses repo git timeout durations', () => { + expect(parseDurationMs('10s')).toBe(10_000); + expect(parseDurationMs('2m')).toBe(120_000); + expect(parseDurationMs('5000ms')).toBe(5000); + expect(parseDurationMs('10')).toBe(10_000); + expect(parseDurationMs(10)).toBe(10_000); + }); + + it('keeps branch compatibility but rejects branch and branches together', async () => { + await fs.writeFile( + path.join(gitnexusHome, 'watch_config.yml'), + [ + 'sync_interval_minutes: 10', + 'projects:', + ' - local_path: /tmp/repos', + ' branch: master', + ' branches: [develop]', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + ].join('\n'), + ); + + const loaded = await loadAutoSyncConfig(); + + expect(loaded.ok).toBe(false); + if (loaded.ok) throw new Error('expected invalid config'); + expect(loaded.message).toContain('must not set both branch and branches'); + }); + + it('uses commit ids to skip unchanged analyses and retry failed prior analyses', () => { + expect(shouldAnalyzeCommit({ currentCommit: 'abc', previousAnalyzedCommit: 'abc' })).toBe( + false, + ); + expect( + shouldAnalyzeCommit({ + currentCommit: 'abc', + previousAnalyzedCommit: 'abc', + previousStatus: 'failed', + }), + ).toBe(true); + expect(shouldAnalyzeCommit({ currentCommit: 'def', previousAnalyzedCommit: 'abc' })).toBe(true); + }); + + it('saves state atomically and reloads it', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + + await saveAutoSyncState( + { + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }, + statePath, + ); + + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('auto-sync-state.json.tmp'), + ); + await expect(loadAutoSyncState(statePath)).resolves.toEqual({ + '/tmp/repos/qts_account|master': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 2, + lastAnalyzeError: 'old error', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }); + }); + + it('returns empty state and reports corrupt state files', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + await fs.writeFile(statePath, '{not-json', 'utf-8'); + + await expect(loadAutoSyncState(statePath)).resolves.toEqual({}); + + expect(stderr).toHaveBeenCalledWith( + `[auto-sync] Ignoring corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + }); + + it('propagates an unreadable state file instead of overwriting it with empty state', async () => { + // A directory stands in for any non-ENOENT read failure (EACCES, EIO). + // Returning {} here would make the next tick persist empty state over + // every repo's analyzed commit and failure counters. + const statePath = path.join(tempDir, 'unreadable-state.json'); + await fs.mkdir(statePath, { recursive: true }); + + await expect(loadAutoSyncState(statePath)).rejects.toThrow(); + }); + + it('drops malformed state entries while preserving valid entries', async () => { + const statePath = path.join(tempDir, 'auto-sync-state.json'); + await fs.writeFile( + statePath, + JSON.stringify({ + '/tmp/repos/valid|main': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 0, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + '/tmp/repos/invalid|main': { + codeCommitId: 123, + analyzeConsecutiveFailures: -1, + lastSyncTime: null, + }, + }), + ); + + await expect(loadAutoSyncState(statePath)).resolves.toEqual({ + '/tmp/repos/valid|main': { + codeCommitId: 'abc', + analyzedCommitId: 'abc', + lastAnalyzeStatus: 'success', + analyzeConsecutiveFailures: 0, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + }); + }); + + it('writes project_commit_info.txt atomically', async () => { + const infoPath = path.join(tempDir, 'project_commit_info.txt'); + + await writeProjectCommitInfo( + [ + { + remoteUrl: 'git@github.com:owner/repo.git', + localPath: '/tmp/repos/repo', + branch: 'master', + codeCommitId: 'abc', + analyzedCommitId: 'abc', + status: 'success', + analyzeConsecutiveFailures: 0, + analyzeFailureThreshold: 3, + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + { + remoteUrl: 'git@github.com:owner/bad.git', + localPath: '/tmp/repos/bad', + branch: 'master', + codeCommitId: 'def', + analyzedCommitId: 'abc', + status: 'threshold_skipped', + analyzeConsecutiveFailures: 3, + analyzeFailureThreshold: 3, + lastAnalyzeError: 'parser crashed', + lastSyncTime: '2026-06-30T00:00:00.000Z', + }, + ], + infoPath, + ); + + const content = await fs.readFile(infoPath, 'utf-8'); + expect(content).toContain('remote: git@github.com:owner/repo.git'); + expect(content).toContain('code_commit: abc'); + expect(content).toContain('analyze_consecutive_failures: 0'); + expect(content).toContain('analyze_failure_threshold: 3'); + expect(content).toContain('status: threshold_skipped'); + expect(content).toContain('last_analyze_error: parser crashed'); + await expect(fs.readdir(tempDir)).resolves.not.toContain( + expect.stringContaining('project_commit_info.txt.tmp'), + ); + }); +}); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index a35541d18..e414c6cb6 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -1,5 +1,6 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { Command, Option } from 'commander'; @@ -16,7 +17,11 @@ function runHelp(command: string, env: NodeJS.ProcessEnv = {}) { } function runHelpArgs(args: string[], env: NodeJS.ProcessEnv = {}) { - return spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, ...args, '--help'], { + return runCliArgs([...args, '--help'], env); +} + +function runCliArgs(args: string[], env: NodeJS.ProcessEnv = {}) { + return spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, ...args], { cwd: repoRoot, encoding: 'utf8', env: { ...process.env, ...env }, @@ -242,6 +247,110 @@ describe('CLI help surface', () => { } }); + it('auto-sync help exposes lifecycle actions and state files', () => { + const result = runHelp('auto-sync'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('gitnexus auto-sync [options] [action]'); + expect(result.stdout).toContain('Actions: init, start (default), restart, stop, status, reset'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch_config.yml'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/watch.pid'); + expect(result.stdout).toContain('GITNEXUS_HOME/watch/project_commit_info.txt'); + }); + + it('watch is reserved and does not start auto-sync or local watch', () => { + const help = runHelp('watch'); + expect(help.status).toBe(0); + expect(help.stdout).toContain('gitnexus watch [options] [action]'); + expect(help.stdout).toContain('gitnexus analyze --watch'); + expect(help.stdout).toContain('gitnexus auto-sync start'); + expect(help.stdout).not.toContain('GITNEXUS_HOME/watch_config.yml'); + + const started = runCliArgs(['watch'], {}); + expect(started.status).toBe(1); + expect(started.stderr).toContain('gitnexus watch'); + expect(started.stderr).toContain('gitnexus analyze --watch'); + expect(started.stderr).toContain('gitnexus auto-sync start'); + + const startAction = runCliArgs(['watch', 'start'], {}); + expect(startAction.status).toBe(1); + expect(startAction.stderr).toContain('gitnexus auto-sync start'); + }); + + it('auto-sync init creates the default watch_config.yml and does not overwrite it', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-init-')); + try { + const first = runCliArgs(['auto-sync', 'init'], { GITNEXUS_HOME: home }); + const configPath = path.join(home, 'watch_config.yml'); + + expect(first.status).toBe(0); + expect(first.stdout).toContain(`Created ${configPath}`); + const config = fs.readFileSync(configPath, 'utf8'); + expect(config).toContain('sync_interval_minutes: 10'); + expect(config).toContain('analyze_failure_threshold: 3'); + expect(config).toContain('analyze_timeout: 5m'); + expect(config).toContain('overwrite_local_changes: false'); + expect(config).toContain(`local_path: ${path.join(home, 'repos')}`); + expect(config).not.toContain('/abs/path/to/repos'); + expect(config).toContain('git@github.com:owner/repo.git'); + expect(config).not.toContain('group_name:'); + + const second = runCliArgs(['auto-sync', 'init'], { GITNEXUS_HOME: home }); + + expect(second.status).toBe(1); + expect(second.stderr).toContain(`Config already exists: ${configPath}`); + expect(fs.readFileSync(configPath, 'utf8')).toBe(config); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it('auto-sync reset removes only derived auto-sync state files', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-reset-')); + const watchDir = path.join(home, 'watch'); + const cloneMarker = path.join(home, 'repos', 'repo', 'keep.txt'); + try { + fs.mkdirSync(path.dirname(cloneMarker), { recursive: true }); + fs.writeFileSync(cloneMarker, 'keep'); + fs.mkdirSync(watchDir, { recursive: true }); + fs.writeFileSync(path.join(watchDir, 'auto-sync-state.json'), '{}'); + fs.writeFileSync(path.join(watchDir, 'project_commit_info.txt'), 'derived'); + + const result = runCliArgs(['auto-sync', 'reset'], { GITNEXUS_HOME: home }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('Reset analysis state'); + expect(fs.existsSync(path.join(watchDir, 'auto-sync-state.json'))).toBe(false); + expect(fs.existsSync(path.join(watchDir, 'project_commit_info.txt'))).toBe(false); + expect(fs.readFileSync(cloneMarker, 'utf8')).toBe('keep'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it('auto-sync stop exits non-zero when no watch was stopped', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-stop-')); + try { + const result = runCliArgs(['auto-sync', 'stop'], { GITNEXUS_HOME: home }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('Watch is not running'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it('auto-sync restart starts when the watch is not running', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-watch-restart-')); + try { + const result = runCliArgs(['auto-sync', 'restart'], { GITNEXUS_HOME: home }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('Watch is not running'); + expect(result.stderr).toContain('Missing config file'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it('wiki help shows provider, review, and verbose flags', () => { const result = runHelp('wiki'); diff --git a/gitnexus/test/unit/file-lock.test.ts b/gitnexus/test/unit/file-lock.test.ts new file mode 100644 index 000000000..4f43ac778 --- /dev/null +++ b/gitnexus/test/unit/file-lock.test.ts @@ -0,0 +1,227 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { acquireFileLock, FileLockBusyError } from '../../src/storage/file-lock.js'; + +const tempDirs: string[] = []; + +async function tempLockPath(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-file-lock-')); + tempDirs.push(dir); + return path.join(dir, 'locks', 'test.mutex'); +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +describe('file lock', () => { + it('rejects a second holder for the same path', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + + await expect(acquireFileLock(lockPath)).rejects.toBeInstanceOf(FileLockBusyError); + + await release(); + }); + + it('propagates hard-link EPERM when no lock exists', async () => { + const lockPath = await tempLockPath(); + const error = Object.assign(new Error('hard links unavailable'), { code: 'EPERM' }); + const link = vi.spyOn(fs, 'link').mockRejectedValueOnce(error); + + try { + await expect(acquireFileLock(lockPath)).rejects.toBe(error); + } finally { + link.mockRestore(); + } + }); + + it('releases idempotently', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + + await release(); + await expect(release()).resolves.toBeUndefined(); + const nextRelease = await acquireFileLock(lockPath); + await nextRelease(); + }); + + it('reclaims a lock whose owner process exited', async () => { + const lockPath = await tempLockPath(); + const oldRelease = await acquireFileLock(lockPath, { + pid: 111, + processStartTime: 'old-start', + }); + + const nextRelease = await acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'new-start', + isProcessAlive: () => false, + }); + + await oldRelease(); + await expect( + acquireFileLock(lockPath, { + isProcessAlive: (pid) => pid === 222, + readProcessStartTime: () => 'new-start', + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await nextRelease(); + }); + + it('reclaims a reused pid only when its start time differs', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 111, processStartTime: 'old-start' }); + + await expect( + acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'next-start', + isProcessAlive: () => true, + readProcessStartTime: () => 'old-start', + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + + const nextRelease = await acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'next-start', + isProcessAlive: () => true, + readProcessStartTime: () => 'reused-pid-start', + }); + + await nextRelease(); + }); + + it('never reclaims a lock held from another host', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { + pid: 111, + processStartTime: 'peer-start', + hostname: 'peer-host', + }); + + // Locally pid 111 is alive with a different start time, which is the + // reuse signature — but the holder is on another machine, so this kernel + // cannot judge it and the lock must stand. + await expect( + acquireFileLock(lockPath, { + pid: 222, + processStartTime: 'local-start', + hostname: 'local-host', + isProcessAlive: () => true, + readProcessStartTime: () => 'unrelated-local-start', + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + }); + + it('fails closed for legacy or invalid lock contents without owner metadata', async () => { + const lockPath = await tempLockPath(); + const invalidContents = ['legacy lock', '{not json', JSON.stringify({ pid: 123 })]; + await fs.mkdir(path.dirname(lockPath), { recursive: true }); + + for (const content of invalidContents) { + await fs.writeFile(lockPath, content, 'utf-8'); + await expect( + acquireFileLock(lockPath, { pid: 456, processStartTime: 'next-start' }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.readFile(lockPath, 'utf-8')).resolves.toBe(content); + await fs.rm(lockPath); + } + + await fs.mkdir(lockPath, { recursive: true }); + await expect( + acquireFileLock(lockPath, { pid: 456, processStartTime: 'next-start' }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.access(lockPath)).resolves.toBeUndefined(); + }); + + it('recovers when a stale reclaim guard was left by a crashed contender', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 999, processStartTime: 'abandoned' }); + await acquireFileLock(`${lockPath}.reclaim`, { + pid: 998, + processStartTime: 'abandoned-reclaimer', + }); + + const release = await acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }); + + await release(); + }); + + it('waits for the current holder when retries are configured', async () => { + const lockPath = await tempLockPath(); + const release = await acquireFileLock(lockPath); + const next = acquireFileLock(lockPath, { retries: 20, retryDelayMs: 5 }); + + await sleep(10); + await release(); + const nextRelease = await next; + + await nextRelease(); + }); + + it('fails closed while another stale-lock recovery is in progress', async () => { + const lockPath = await tempLockPath(); + const oldRelease = await acquireFileLock(lockPath, { + pid: 999, + processStartTime: 'abandoned', + }); + const reclaimGuardPath = `${lockPath}.reclaim`; + await fs.mkdir(reclaimGuardPath); + + await expect( + acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }), + ).rejects.toBeInstanceOf(FileLockBusyError); + await expect(fs.access(lockPath)).resolves.toBeUndefined(); + + await fs.rmdir(reclaimGuardPath); + const nextRelease = await acquireFileLock(lockPath, { + pid: 1000, + processStartTime: 'next', + isProcessAlive: () => false, + }); + await oldRelease(); + await nextRelease(); + }); + + it('allows only one contender to recover an abandoned lock', async () => { + const lockPath = await tempLockPath(); + await acquireFileLock(lockPath, { pid: 999, processStartTime: 'abandoned' }); + const starts = new Map( + Array.from({ length: 8 }, (_, index) => [1000 + index, `start-${index}`]), + ); + + const results = await Promise.allSettled( + [...starts].map(([pid, processStartTime]) => + acquireFileLock(lockPath, { + pid, + processStartTime, + isProcessAlive: (ownerPid) => ownerPid !== 999, + readProcessStartTime: (ownerPid) => starts.get(ownerPid), + }), + ), + ); + + const acquired = results.filter( + (result): result is PromiseFulfilledResult<() => Promise> => + result.status === 'fulfilled', + ); + expect(acquired).toHaveLength(1); + for (const result of results) { + if (result.status === 'rejected') expect(result.reason).toBeInstanceOf(FileLockBusyError); + } + await acquired[0].value(); + }); +}); diff --git a/gitnexus/test/unit/git-clone.test.ts b/gitnexus/test/unit/git-clone.test.ts index 4db09c837..0c365e359 100644 --- a/gitnexus/test/unit/git-clone.test.ts +++ b/gitnexus/test/unit/git-clone.test.ts @@ -16,20 +16,24 @@ vi.mock('../../src/core/logger.js', () => ({ import { extractRepoName, + extractWebRepoName, getCloneDir, validateGitUrl, cloneOrPull, buildCloneArgs, + buildBranchCloneArgs, buildGitEnv, normalizeGitUrlForCompare, assertRemoteMatchesRequestedUrl, isAzureDevOpsUrl, warnIfInsecureAzureConfig, + runGitForTest, } from '../../src/server/git-clone.js'; import path from 'node:path'; import os from 'node:os'; import fs from 'node:fs/promises'; import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; import { getRemoteOriginUrl } from '../../src/storage/git.js'; import { getGlobalDir } from '../../src/storage/repo-manager.js'; @@ -42,6 +46,31 @@ import { getGlobalDir } from '../../src/storage/repo-manager.js'; // load, the same point CLONE_ROOT is frozen, so the two always agree. const EXPECTED_CLONE_ROOT = path.resolve(path.join(getGlobalDir(), 'repos')); +async function mkControlledRoot(prefix: string): Promise { + const base = path.join(process.cwd(), '.tmp-test'); + await fs.mkdir(base, { recursive: true }); + return fs.realpath(await fs.mkdtemp(path.join(base, prefix))); +} + +function runGit(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const proc = spawn('git', args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk; + }); + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk; + }); + proc.on('close', (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(`git ${args.join(' ')} failed (${code}): ${stderr}`)); + }); + proc.on('error', reject); + }); +} + describe('git-clone', () => { describe('extractRepoName', () => { it('extracts name from HTTPS URL', () => { @@ -95,29 +124,39 @@ describe('git-clone', () => { expect(elapsedMs).toBeLessThan(500); }); - it('strips leading dashes to prevent argument injection', () => { - expect(extractRepoName('https://github.com/user/--upload-pack=payload.git')).toBe( - 'upload-pack_payload', + it('rejects leading dashes to prevent argument injection', () => { + expect(() => extractRepoName('https://github.com/user/--upload-pack=payload.git')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://github.com/user/-repo')).toThrow( + 'valid repository name', ); - expect(extractRepoName('https://github.com/user/-repo')).toBe('repo'); }); - it('sanitizes unsafe directory characters', () => { - // sanitizeRepoName turns into _tag_ - expect(extractRepoName('https://github.com/user/repo.git')).toBe('repo_tag_'); + it('rejects unsafe directory characters instead of sanitizing them', () => { + expect(() => extractRepoName('https://github.com/user/repo.git')).toThrow( + 'valid repository name', + ); }); - it('sanitizes shell metacharacters in URL segments', () => { + it('rejects shell metacharacters in URL segments', () => { // The split on /[/:]/ does not split on backslashes or other shell chars, - // so a name like `repo;rm -rf /` would slip through without the pattern. - // After fix/sanitize-repo-name, these are sanitized to underscores. - expect(extractRepoName('https://example.com/foo:repo;rm')).toBe('repo_rm'); - expect(extractRepoName('https://example.com/foo:repo$x')).toBe('repo_x'); + // so a name like `repo;rm -rf /` must fail instead of being rewritten. + expect(() => extractRepoName('https://example.com/foo:repo;rm')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo$x')).toThrow( + 'valid repository name', + ); }); - it('sanitizes whitespace and backslashes', () => { - expect(extractRepoName('https://example.com/foo:repo name')).toBe('repo_name'); - expect(extractRepoName('https://example.com/foo:repo\\name')).toBe('repo_name'); + it('rejects whitespace and backslashes', () => { + expect(() => extractRepoName('https://example.com/foo:repo name')).toThrow( + 'valid repository name', + ); + expect(() => extractRepoName('https://example.com/foo:repo\\name')).toThrow( + 'valid repository name', + ); }); }); @@ -158,6 +197,15 @@ describe('git-clone', () => { expect(() => validateGitUrl('http://gitlab.com/user/repo.git')).not.toThrow(); }); + it('rejects query strings and fragments instead of reinterpreting clone remotes', () => { + expect(() => validateGitUrl('https://github.com/user/repo.git?ref=main')).toThrow( + 'must not include query strings or fragments', + ); + expect(() => validateGitUrl('https://github.com/user/repo.git#main')).toThrow( + 'must not include query strings or fragments', + ); + }); + it('blocks SSH protocol', () => { expect(() => validateGitUrl('ssh://git@github.com/user/repo.git')).toThrow( 'Only https:// and http://', @@ -349,9 +397,23 @@ describe('git-clone', () => { expect(args.some((a) => a.toLowerCase().includes('authorization'))).toBe(false); expect(args.some((a) => a.includes('extraHeader'))).toBe(false); }); + + it('adds --branch before the URL separator for branch-specific clones', () => { + const args = buildBranchCloneArgs('git@github.com:owner/repo.git', '/safe/target', 'develop'); + expect(args).toEqual([ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@github.com:owner/repo.git', + '/safe/target', + ]); + }); }); - describe('buildGitEnv — token injection', () => { + describe('buildGitEnv — managed git environment', () => { // The token MUST travel via GIT_CONFIG_* env vars (git ≥2.31), not via // argv or URL. This keeps it out of `ps`, shell history, and stderr. @@ -375,33 +437,33 @@ describe('git-clone', () => { expect(env.GIT_CURL_VERBOSE).toBeUndefined(); }); - it('does not set GIT_CONFIG_* env vars when no token is provided', () => { + it('disables repository hooks even when no token is provided', () => { const env = buildGitEnv({}); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); - expect(env.GIT_CONFIG_KEY_0).toBeUndefined(); - expect(env.GIT_CONFIG_VALUE_0).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_KEY_0).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_0).toBe(os.devNull); }); - it('also leaves GIT_CONFIG_* unset when token is empty string', () => { + it('only disables repository hooks when token is empty string', () => { const env = buildGitEnv({}, { token: '' }); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); - expect(env.GIT_CONFIG_KEY_0).toBeUndefined(); - expect(env.GIT_CONFIG_VALUE_0).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_KEY_0).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_0).toBe(os.devNull); }); it('injects a host-scoped Basic-auth header when a github.com token is provided', () => { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://github.com/owner/repo' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); + expect(env.GIT_CONFIG_COUNT).toBe('2'); // Host-scoped key: the header attaches only to this origin's requests. - expect(env.GIT_CONFIG_KEY_0).toBe('http.https://github.com/owner/repo.extraHeader'); + expect(env.GIT_CONFIG_KEY_1).toBe('http.https://github.com/owner/repo.extraHeader'); const expected = 'Authorization: Basic ' + Buffer.from('x-access-token:ghp_secret123').toString('base64'); - expect(env.GIT_CONFIG_VALUE_0).toBe(expected); + expect(env.GIT_CONFIG_VALUE_1).toBe(expected); }); it('does not inject a token for a non-github host (defense-in-depth host bind)', () => { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://gitlab.com/owner/repo' }); - expect(env.GIT_CONFIG_COUNT).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('1'); }); it('never includes the raw token value in any env entry', () => { @@ -410,7 +472,7 @@ describe('git-clone', () => { const token = 'ghp_uniqueRawSecret_98765'; const env = buildGitEnv({ EXISTING: 'value' }, { token, url: 'https://github.com/o/r' }); for (const [key, value] of Object.entries(env)) { - if (key === 'GIT_CONFIG_VALUE_0') continue; + if (key === 'GIT_CONFIG_VALUE_1') continue; expect(String(value)).not.toContain(token); } }); @@ -420,12 +482,12 @@ describe('git-clone', () => { process.env.AZURE_DEVOPS_PAT = 'azure-pat-xyz'; try { const env = buildGitEnv({}, { url: 'https://dev.azure.com/org/proj/_git/repo' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); - expect(env.GIT_CONFIG_KEY_0).toBe( + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_KEY_1).toBe( 'http.https://dev.azure.com/org/proj/_git/repo.extraHeader', ); const expected = 'Authorization: Basic ' + Buffer.from(':azure-pat-xyz').toString('base64'); - expect(env.GIT_CONFIG_VALUE_0).toBe(expected); + expect(env.GIT_CONFIG_VALUE_1).toBe(expected); } finally { if (prev === undefined) delete process.env.AZURE_DEVOPS_PAT; else process.env.AZURE_DEVOPS_PAT = prev; @@ -439,8 +501,8 @@ describe('git-clone', () => { process.env.AZURE_DEVOPS_PAT = 'azure-pat-xyz'; try { const env = buildGitEnv({}, { token: 'ghp_secret123', url: 'https://github.com/o/r' }); - expect(env.GIT_CONFIG_COUNT).toBe('1'); - expect(env.GIT_CONFIG_VALUE_1).toBeUndefined(); + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_VALUE_2).toBeUndefined(); for (const value of Object.values(env)) { expect(String(value)).not.toContain('azure-pat-xyz'); } @@ -455,13 +517,48 @@ describe('git-clone', () => { { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.sslVerify', GIT_CONFIG_VALUE_0: 'true' }, { token: 'ghp_secret123', url: 'https://github.com/o/r' }, ); - expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_COUNT).toBe('3'); // Operator's pre-existing config is preserved at index 0. expect(env.GIT_CONFIG_KEY_0).toBe('http.sslVerify'); expect(env.GIT_CONFIG_VALUE_0).toBe('true'); - // Our credential is appended at index 1. - expect(env.GIT_CONFIG_KEY_1).toBe('http.https://github.com/o/r.extraHeader'); - expect(env.GIT_CONFIG_VALUE_1).toContain('Authorization: Basic '); + expect(env.GIT_CONFIG_KEY_1).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_1).toBe(os.devNull); + expect(env.GIT_CONFIG_KEY_2).toBe('http.https://github.com/o/r.extraHeader'); + expect(env.GIT_CONFIG_VALUE_2).toContain('Authorization: Basic '); + }); + + it('overrides an inherited hooks path with the managed safe value', () => { + const env = buildGitEnv({ + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.hooksPath', + GIT_CONFIG_VALUE_0: '/tmp/untrusted-hooks', + }); + expect(env.GIT_CONFIG_COUNT).toBe('2'); + expect(env.GIT_CONFIG_KEY_1).toBe('core.hooksPath'); + expect(env.GIT_CONFIG_VALUE_1).toBe(os.devNull); + }); + + it('does not execute hooks from an existing repository', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-managed-git-'); + const marker = path.join(root, 'hook-ran'); + try { + await runGit(['init', '--initial-branch=main'], root); + await runGit(['config', 'user.email', 'test@example.com'], root); + await runGit(['config', 'user.name', 'GitNexus Test'], root); + await fs.writeFile(path.join(root, 'README.md'), 'test\n'); + await runGit(['add', 'README.md'], root); + await runGit(['commit', '-m', 'initial'], root); + const hook = path.join(root, '.git', 'hooks', 'post-checkout'); + await fs.writeFile(hook, `#!/bin/sh\ntouch ${JSON.stringify(marker)}\n`); + await fs.chmod(hook, 0o700); + + await runGitForTest(['checkout', '-b', 'next'], root); + + await expect(fs.access(marker)).rejects.toThrow(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } }); it('strips control characters from the config key (no key injection)', () => { @@ -469,7 +566,7 @@ describe('git-clone', () => { {}, { token: 'ghp_secret123', url: 'https://github.com/o/r%0Anewline' }, ); - const key = env.GIT_CONFIG_KEY_0 ?? ''; + const key = env.GIT_CONFIG_KEY_1 ?? ''; expect(key).not.toContain('\n'); expect(key).not.toContain('\r'); }); @@ -534,6 +631,387 @@ describe('git-clone', () => { 'Only https:// and http://', ); }); + + it('keeps regular cloneOrPull restricted to http and https URLs', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('git@github.com:owner/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('Invalid URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('creates missing nested parents before checking controlled clone containment', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'github.com', 'owner', 'repo'); + const runGitForTest = vi.fn(async () => { + await fs.mkdir(path.join(target, '.git'), { recursive: true }); + return ''; + }); + try { + await expect( + cloneOrPull('git@github.com:owner/repo', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + runGitForTest, + }), + ).resolves.toBe(target); + + expect(runGitForTest).toHaveBeenCalledOnce(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('allows auto-sync SSH SCP clone URLs with a per-repo timeout', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const runGitForTest = vi.fn(async () => { + await fs.mkdir(target); + return ''; + }); + try { + await expect( + cloneOrPull('git@gitlab.com:group/subgroup/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + timeoutMs: 10_000, + branch: 'develop', + runGitForTest, + }), + ).resolves.toBe(target); + + expect(runGitForTest).toHaveBeenCalledWith( + [ + 'clone', + '--depth', + '1', + '--branch', + 'develop', + '--', + 'git@gitlab.com:group/subgroup/repo.git', + target, + ], + undefined, + { token: undefined, url: 'git@gitlab.com:group/subgroup/repo.git', timeoutMs: 10_000 }, + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('allows an explicitly controlled auto-sync clone root outside the default root', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + const target = path.join(root, 'repo'); + await expect( + cloneOrPull('http://127.0.0.1/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('private/internal'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled-root target names that do not match the remote repo name', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'other'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('basename must match'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects symlink children before clone or pull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const outside = await mkControlledRoot('gitnexus-outside-'); + try { + await fs.symlink(outside, path.join(root, 'repo')); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('symlink'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects writable existing directories below a controlled clone root', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const namespace = path.join(root, 'team'); + try { + await fs.mkdir(namespace); + await fs.chmod(namespace, 0o777); + await expect( + cloneOrPull( + 'https://example.com/team/repo.git', + path.join(namespace, 'repo'), + undefined, + { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }, + ), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(namespace, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects writable .git metadata in an existing controlled clone', async () => { + if (process.platform === 'win32') return; + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const gitDir = path.join(target, '.git'); + try { + await fs.mkdir(gitDir, { recursive: true }); + await fs.chmod(gitDir, 0o777); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(gitDir, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked .git metadata in an existing controlled clone', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const outside = await mkControlledRoot('gitnexus-outside-git-dir-'); + const target = path.join(root, 'repo'); + try { + await fs.mkdir(target); + await fs.symlink(outside, path.join(target, '.git')); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('symlink'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects existing clones whose remote origin mismatches the requested URL', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + try { + await new Promise((resolve, reject) => { + const proc = spawn('git', ['init'], { cwd: root, stdio: 'ignore' }); + proc.on('close', (code) => + code === 0 ? resolve() : reject(new Error(`git init ${code}`)), + ); + proc.on('error', reject); + }); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')).catch(async () => { + await fs.mkdir(target); + await fs.rename(path.join(root, '.git'), path.join(target, '.git')); + }); + await fs.writeFile( + path.join(target, '.git', 'config'), + [ + '[remote "origin"]', + '\turl = https://example.com/other/repo.git', + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '', + ].join('\n'), + ); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('not the requested URL'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('switches a shallow single-branch clone to a fallback branch', async () => { + const root = await mkControlledRoot('gitnexus-shallow-fallback-'); + const source = path.join(root, 'source'); + const remote = path.join(root, 'remote.git'); + const target = path.join(root, 'repo'); + const remoteUrl = 'git@github.com:team/repo.git'; + const gitConfig = path.join(root, 'gitconfig'); + const previousGlobalConfig = process.env.GIT_CONFIG_GLOBAL; + const previousNoSystemConfig = process.env.GIT_CONFIG_NOSYSTEM; + + try { + await runGit(['init', '--bare', remote], root); + await runGit(['init', '--initial-branch=master', source], root); + await runGit(['config', 'user.email', 'test@example.com'], source); + await runGit(['config', 'user.name', 'GitNexus Test'], source); + await fs.writeFile(path.join(source, 'branch.txt'), 'master\n'); + await runGit(['add', 'branch.txt'], source); + await runGit(['commit', '-m', 'master'], source); + await runGit(['checkout', '-b', 'main'], source); + await fs.writeFile(path.join(source, 'branch.txt'), 'main\n'); + await runGit(['commit', '-am', 'main'], source); + await runGit(['remote', 'add', 'origin', `file://${remote}`], source); + await runGit(['push', 'origin', 'master', 'main'], source); + + await fs.writeFile( + gitConfig, + `[protocol "file"]\n\tallow = always\n[url "file://${remote}"]\n\tinsteadOf = ${remoteUrl}\n`, + ); + process.env.GIT_CONFIG_GLOBAL = gitConfig; + process.env.GIT_CONFIG_NOSYSTEM = '1'; + + await runGit(['clone', '--depth', '1', '--branch', 'master', remoteUrl, target], root); + await expect( + runGit(['show-ref', '--verify', '--quiet', 'refs/remotes/origin/main'], target), + ).rejects.toThrow(); + await expect(runGit(['rev-parse', '--is-shallow-repository'], target)).resolves.toBe( + 'true\n', + ); + + await fs.writeFile(path.join(target, 'branch.txt'), 'local changes\n'); + await expect( + cloneOrPull(remoteUrl, target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + branch: 'main', + }), + ).rejects.toThrow(); + await expect(fs.readFile(path.join(target, 'branch.txt'), 'utf8')).resolves.toBe( + 'local changes\n', + ); + + await cloneOrPull(remoteUrl, target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + allowAutoSyncSsh: true, + branch: 'main', + overwriteLocalChanges: true, + }); + + await expect(runGit(['branch', '--show-current'], target)).resolves.toBe('main\n'); + await expect(fs.readFile(path.join(target, 'branch.txt'), 'utf8')).resolves.toBe('main\n'); + await expect(runGit(['rev-parse', 'main'], target)).resolves.toBe( + await runGit(['rev-parse', 'origin/main'], target), + ); + } finally { + if (previousGlobalConfig === undefined) delete process.env.GIT_CONFIG_GLOBAL; + else process.env.GIT_CONFIG_GLOBAL = previousGlobalConfig; + if (previousNoSystemConfig === undefined) delete process.env.GIT_CONFIG_NOSYSTEM; + else process.env.GIT_CONFIG_NOSYSTEM = previousNoSystemConfig; + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('clones into a pre-existing empty target directory', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const target = path.join(root, 'repo'); + const runGitForTest = vi.fn(async () => ''); + try { + await fs.mkdir(target); + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + runGitForTest, + }), + ).resolves.toBe(target); + expect(runGitForTest).toHaveBeenCalled(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('quarantines partial auto-sync clone output on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + runGitForTest: async () => { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'partial.txt'), 'partial', 'utf-8'); + throw new Error('git clone failed (exit code 128)'); + }, + }), + ).rejects.toThrow('git clone failed'); + + const entries = await fs.readdir(quarantineRoot); + expect( + entries.some((entry) => entry.startsWith('auto-sync-') && entry.endsWith('-repo')), + ).toBe(true); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('does not quarantine an existing non-git directory on clone failure', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + const quarantineRoot = path.join(root, 'quarantine'); + const target = path.join(root, 'repo'); + try { + await fs.mkdir(target); + await fs.writeFile(path.join(target, 'user-file.txt'), 'keep me', 'utf-8'); + + await expect( + cloneOrPull('https://example.com/team/repo.git', target, undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + quarantineRoot, + }), + ).rejects.toThrow('already exists but is not a git repository'); + + await expect(fs.readFile(path.join(target, 'user-file.txt'), 'utf-8')).resolves.toBe( + 'keep me', + ); + await expect(fs.access(quarantineRoot)).rejects.toThrow(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects controlled clone roots with unsafe permissions inside cloneOrPull', async () => { + const root = await mkControlledRoot('gitnexus-controlled-root-'); + try { + await fs.chmod(root, 0o777); + await expect( + cloneOrPull('https://example.com/team/repo.git', path.join(root, 'repo'), undefined, { + allowedCloneRoot: root, + expectedRepoName: 'repo', + }), + ).rejects.toThrow('world-writable'); + } finally { + await fs.chmod(root, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + } + }); }); describe('isAzureDevOpsUrl', () => { @@ -646,6 +1124,30 @@ describe('git-clone', () => { }); }); + describe('extractWebRepoName — API clone compatibility', () => { + it('sanitizes repo names with spaces and unsafe directory characters at the web boundary', () => { + expect(extractWebRepoName('https://dev.azure.com/org/project/_git/My Repo With Spaces')).toBe( + 'My_Repo_With_Spaces', + ); + expect(extractWebRepoName('https://example.com/team/repo$name.git')).toBe('repo_name'); + }); + + it('keeps Windows reserved names from becoming clone directories', () => { + expect(() => extractWebRepoName('https://example.com/team/CON.git')).toThrow( + 'valid repository name', + ); + expect(() => extractWebRepoName('https://example.com/team/NUL.txt')).toThrow( + 'valid repository name', + ); + }); + + it('leaves strict extractRepoName behavior unchanged for internal callers', () => { + expect(() => extractRepoName('https://example.com/team/repo$name.git')).toThrow( + 'valid repository name', + ); + }); + }); + describe('validateGitUrl — Azure DevOps URLs', () => { it('allows self-hosted Azure DevOps Server URLs', () => { expect(() => @@ -824,4 +1326,42 @@ describe('git-clone', () => { } }); }); + + describe('runGit timeout', () => { + it('rejects after SIGKILL even when the child never closes', async () => { + vi.useFakeTimers(); + try { + const child = new EventEmitter() as EventEmitter & { + stderr: EventEmitter; + kill: ReturnType; + }; + child.stderr = new EventEmitter(); + child.kill = vi.fn(); + const spawnForTest = vi.fn(() => child) as unknown as typeof spawn; + + const promise = runGitForTest(['clone'], undefined, { + timeoutMs: 20, + timeoutKillGraceMs: 20, + spawnForTest, + }); + + await vi.advanceTimersByTimeAsync(25); + let settled = false; + promise + .catch(() => {}) + .finally(() => { + settled = true; + }); + await vi.runAllTicks(); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(25); + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + await expect(promise).rejects.toThrow('timed out after 20ms'); + } finally { + vi.useRealTimers(); + } + }); + }); }); diff --git a/gitnexus/test/unit/hooks.test.ts b/gitnexus/test/unit/hooks.test.ts index 062c5b870..54e709233 100644 --- a/gitnexus/test/unit/hooks.test.ts +++ b/gitnexus/test/unit/hooks.test.ts @@ -413,14 +413,32 @@ describe('windowsHide regression', () => { /** * Count spawn-family invocations. The regex matches ``spawn(``, * ``spawnSync(``, ``execFile(``, ``execFileSync(``, - * ``execFileAsync(``, ``execSync(`` as function calls — not - * destructures (``const { spawn } = ...``), not method calls - * (``.exec(``), not bare ``exec()`` (which collides with regex - * ``.exec()``; we explicitly drop it). + * ``execFileAsync(``, ``execSync(`` and simple local aliases that + * point at one of those functions as function calls — not destructures + * (``const { spawn } = ...``), not method calls (``.exec(``), not bare + * ``exec()`` (which collides with regex ``.exec()``; we explicitly + * drop it). */ function countSpawnCalls(codeSource: string): number { - const re = - /(^|[^a-zA-Z0-9_$.])(spawn|spawnSync|execFile|execFileSync|execFileAsync|execSync)\s*\(/gm; + const spawnFunctions = [ + 'spawn', + 'spawnSync', + 'execFile', + 'execFileSync', + 'execFileAsync', + 'execSync', + ]; + const spawnNames = new Set(spawnFunctions); + const aliasRe = new RegExp( + `\\bconst\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*[^;\\n]*\\b(?:${spawnFunctions.join('|')})\\b`, + 'g', + ); + let aliasMatch: RegExpExecArray | null; + while ((aliasMatch = aliasRe.exec(codeSource)) !== null) { + spawnNames.add(aliasMatch[1]); + } + + const re = new RegExp(`(^|[^a-zA-Z0-9_$.])(${[...spawnNames].join('|')})\\s*\\(`, 'gm'); let count = 0; while (re.exec(codeSource) !== null) { count++; diff --git a/gitnexus/test/unit/process-identity.test.ts b/gitnexus/test/unit/process-identity.test.ts new file mode 100644 index 000000000..7b31f1d32 --- /dev/null +++ b/gitnexus/test/unit/process-identity.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { isProcessAlive, readProcessStartTime } from '../../src/utils/process-identity.js'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('process identity', () => { + it('treats only ESRCH as a dead process', () => { + const kill = vi.spyOn(process, 'kill'); + kill.mockImplementationOnce(() => { + throw Object.assign(new Error('missing'), { code: 'ESRCH' }); + }); + kill.mockImplementationOnce(() => { + throw Object.assign(new Error('not permitted'), { code: 'EPERM' }); + }); + + expect(isProcessAlive(111)).toBe(false); + expect(isProcessAlive(222)).toBe(true); + }); + + it.skipIf(process.platform === 'win32')( + 'renders the same start time regardless of the ambient timezone', + () => { + const original = process.env.TZ; + try { + process.env.TZ = 'UTC'; + const utc = readProcessStartTime(process.pid); + process.env.TZ = 'Asia/Tokyo'; + const tokyo = readProcessStartTime(process.pid); + + expect(utc).toBeTruthy(); + // A locale/timezone-dependent identity makes a live lock look reused. + expect(tokyo).toBe(utc); + } finally { + if (original === undefined) delete process.env.TZ; + else process.env.TZ = original; + } + }, + ); +}); diff --git a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts index ba0fbe777..3c6c1c77f 100644 --- a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts @@ -173,8 +173,10 @@ describe('writeRegistry — private tmp path per transaction (#2888)', () => { it('keeps serving a validating read when the prune write fails', async () => { await registerRepo(tmpRepoA.dbPath, meta, { name: 'gone' }); fsCtx.renameMock.mockClear(); + // EBUSY is normally retryable, but prune persistence is best-effort and + // must not hold the registry lock through retry backoff. fsCtx.renameMock.mockImplementationOnce(() => - Promise.reject(Object.assign(new Error('mock read-only home'), { code: 'EROFS' })), + Promise.reject(Object.assign(new Error('mock busy registry'), { code: 'EBUSY' })), ); const cap = _captureLogger(); diff --git a/gitnexus/test/unit/repo-manager.test.ts b/gitnexus/test/unit/repo-manager.test.ts index 468096a61..34a91206f 100644 --- a/gitnexus/test/unit/repo-manager.test.ts +++ b/gitnexus/test/unit/repo-manager.test.ts @@ -841,6 +841,25 @@ describe('registerRepo name override + collision guard (#829)', () => { expect(entries[0].name).not.toBe(path.basename(tmpRepoA.dbPath)); }); + it('preserves every concurrent registration', async () => { + const repoPaths = Array.from({ length: 12 }, (_, index) => + path.join(tmpRepoA.dbPath, `concurrent-${index}`), + ); + await Promise.all(repoPaths.map((repoPath) => fs.mkdir(repoPath, { recursive: true }))); + + await Promise.all( + repoPaths.map((repoPath, index) => + registerRepo(repoPath, meta, { name: `concurrent-${index}` }), + ), + ); + + const entries = await listRegisteredRepos(); + expect(entries).toHaveLength(repoPaths.length); + expect(entries.map((entry) => entry.name).sort()).toEqual( + repoPaths.map((_, index) => `concurrent-${index}`).sort(), + ); + }); + it('re-registerRepo on same path without name preserves an existing alias', async () => { await registerRepo(tmpRepoA.dbPath, meta, { name: 'custom-alias' }); // Second call with no opts should keep the alias, not revert to basename. diff --git a/gitnexus/test/unit/watch-command.test.ts b/gitnexus/test/unit/watch-command.test.ts new file mode 100644 index 000000000..90eeda128 --- /dev/null +++ b/gitnexus/test/unit/watch-command.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const autoSync = vi.hoisted(() => ({ + startAutoSyncWatch: vi.fn(), +})); + +vi.mock('../../src/core/auto-sync/index.js', () => ({ + getAutoSyncConfigPath: vi.fn(() => '/tmp/watch_config.yml'), + getAutoSyncMutexPath: vi.fn(() => '/tmp/watch.mutex'), + readAutoSyncWatchStatus: vi.fn(), + resetAutoSyncState: vi.fn(), + startAutoSyncWatch: autoSync.startAutoSyncWatch, + stopAutoSyncWatch: vi.fn(), +})); + +import { autoSyncCommand } from '../../src/cli/auto-sync.js'; + +describe('auto-sync command', () => { + beforeEach(() => vi.clearAllMocks()); + afterEach(() => vi.restoreAllMocks()); + + it('reports foreground stop failures and exits non-zero', async () => { + const stop = vi.fn(async () => { + throw new Error('cleanup failed'); + }); + autoSync.startAutoSyncWatch.mockResolvedValue({ stop }); + let signalHandler: (() => void) | undefined; + vi.spyOn(process, 'once').mockImplementation(((event, listener) => { + if (event === 'SIGTERM') signalHandler = listener as () => void; + return process; + }) as typeof process.once); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + + await autoSyncCommand('start'); + signalHandler?.(); + await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(1)); + + expect(stop).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith('[auto-sync] Failed to stop watch: cleanup failed\n'); + expect(stderr).not.toHaveBeenCalledWith('[auto-sync] Watch stopped.\n'); + }); +}); diff --git a/gitnexus/test/unit/watch-failure-policy.test.ts b/gitnexus/test/unit/watch-failure-policy.test.ts index aa32f0454..96fcfff7c 100644 --- a/gitnexus/test/unit/watch-failure-policy.test.ts +++ b/gitnexus/test/unit/watch-failure-policy.test.ts @@ -7,7 +7,7 @@ vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: vi.fn(), })); -import { shouldStopAfterWatchRefreshFailure } from '../../src/cli/watch.js'; +import { shouldStopAfterWatchRefreshFailure } from '../../src/cli/analyze-watch.js'; describe('watch refresh failure policy', () => { beforeEach(() => analyzeFailureMayHaveMutatedLiveIndex.mockReset()); diff --git a/gitnexus/test/unit/watch-paths.test.ts b/gitnexus/test/unit/watch-paths.test.ts index 397b29de8..22fb9872b 100644 --- a/gitnexus/test/unit/watch-paths.test.ts +++ b/gitnexus/test/unit/watch-paths.test.ts @@ -3,7 +3,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { createWatchIgnorePredicate } from '../../src/config/ignore-service.js'; -import { isRelevantWatchPath, resolveWatchOptions } from '../../src/cli/watch.js'; +import { isRelevantWatchPath, resolveWatchOptions } from '../../src/cli/analyze-watch.js'; import * as git from '../../src/storage/git.js'; vi.mock('../../src/storage/git.js', () => ({ From b9613ee86b4329c5ba5d69190848f0d29fc12d80 Mon Sep 17 00:00:00 2001 From: ChunxueLi <54129170+ChunxueLi@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:59:45 +0800 Subject: [PATCH 03/21] feat(node): wrapped-client HTTP consumers + leading-prefix template stripping (#3111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(http-patterns): Patch 8 - extract enterprise wrapped-client HTTP consumers Recognize the enterprise axios-wrapper call shape X.request({ url, method }) (e.g. httpClient.request from @winex-plugin/win-request) as a consumer contract source, and strip the leading gateway/service-prefix template variable so consumer paths align with backend provider routes. Effect on sr-next group: 1740 contracts / 0 cross-links -> 3540 contracts / 883 exact cross-links (14 frontend repos <-> backend/opt). Not submitted upstream yet; see CUSTOM_PATCHES.md Patch 8 for details. Co-Authored-By: Claude * feat(node): wrapped-client HTTP consumers + leading-prefix template stripping Consumer extraction for enterprise axios-wrapped clients: - X.request({ url, method }) member form (httpClient.request from win-request and friends): any .request(options) call with url/method|type string props; the url template literal is split on ${...} spans and the longest /-leading literal segment is kept. - Leading ${...} gateway/service-prefix variables are stripped as consumer-path normalization semantics in normalizeHttpPath (stripLeadingTemplatePrefix): `${client}/api/v1/x` → /api/v1/x for fetch/axios/wrapped shapes alike. Mid/tail interpolations still round-trip through {param}; a stripped remainder not starting with / is dropped (same rejection static relative urls get at scan time). - %7B/%7D unescaping for absolute-URL branches. On our 16-repo frontend monorepo this took group sync from 1740 contracts / 0 cross-links to 5111 / 2097 (exact links, 16/16 repos linked). * fix(route): preserve upstream symbol-resolution machinery; strip only Rebasing correction: an earlier iteration of this change simplified the symbol probing to a bare line-1 offset and dropped the exact-module match, which mis-attributed data-table handlers to decoy same-name symbols (6 data-route-table regressions). Restore the upstream probing (toZeroBasedLine + exact-module resolution) wholesale; the deltas this change actually needs are the pure ones: stripLeadingTemplatePrefix, normalizeConsumerPath returning null for un-reducible urls (callers drop), and the %7B/%7D brace restoration in the absolute-URL branch. * style: prettier * Address PR review feedback (#3111) Pass wrapped-client URLs through shared consumer-path normalization instead of the longest-segment reducer, emit * for present-but-non-literal methods, restore {param} via a sentinel so literal %7B segments stay encoded, and drop unused decorator helpers. Co-authored-by: Cursor * Address PR review feedback (#3111) Gate wrapped X.request({url}) on axios-proven receivers or a small wrapper allowlist so cy.request/queue.request cannot mint HTTP consumers. Co-authored-by: Cursor * Address PR review feedback (#3111) Use a private-use sentinel so literal path segments are not rewritten, trim wrapped URLs before the scan-time path gate, and treat interpolated/shorthand methods as *. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command * fix(http): honest wrapped-request methods and tighter admission (#3111) Quoted keys and object spreads were minted as GET; drop spelling-only `api`, align gateway-prefix member verbs with prefix-strip, and keep absolute URLs. Co-authored-by: Cursor --------- Co-authored-by: l.cx Co-authored-by: Claude Co-authored-by: ChunxueLi Co-authored-by: Gergő Magyar Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../group/extractors/http-patterns/node.ts | 155 +++++++++++++- .../group/extractors/http-route-extractor.ts | 63 +++++- .../unit/group/http-route-extractor.test.ts | 111 ++++++++++ .../group/js-http-consumer-resolution.test.ts | 195 ++++++++++++++++++ 4 files changed, 511 insertions(+), 13 deletions(-) diff --git a/gitnexus/src/core/group/extractors/http-patterns/node.ts b/gitnexus/src/core/group/extractors/http-patterns/node.ts index 7a198f595..9edfd69eb 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/node.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/node.ts @@ -13,6 +13,7 @@ import type { HttpDetection, HttpLanguagePlugin, RepoContext } from './types.js' import { MAX_FOLD_LENGTH } from '../../../ingestion/route-extractors/constant-resolver.js'; import { DATA_ROUTE_TABLE_SOURCE, + propertyName, scanDataRouteTables, } from '../../../ingestion/route-extractors/data-route-table.js'; import { extractNestRoutes } from '../../../ingestion/route-extractors/nest.js'; @@ -152,6 +153,37 @@ const AXIOS_OBJECT_SPEC: PatternSpec> = { `, }; +// ─── Consumer: wrapped client X.request({ url, method }) ──────────── +// Enterprise wrapper shape: an axios instance (or a named request helper) +// re-exported under a local name — `httpClient.request({ url, method })` +// from `@winex-plugin/win-request`, `$http.request(...)`. Generic names +// like `api` need axios.create/import proof (`isHttpClientRef`); spelling +// alone is too common (graphql-request helpers, domain `api` objects). +// The member property is `request` (not an HTTP verb), so this cannot +// collide with the Express provider pattern (`router.get`) or the axios +// member form (`axios.get`). Option keys are resolved programmatically, +// same as the jQuery ajax / axios object forms. +// +// The query captures the receiver so scan can reject unrelated +// `.request({ url })` APIs (`cy.request`, `queue.request`). +const REQUEST_OBJECT_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (_) @obj + property: (property_identifier) @fn (#eq? @fn "request")) + arguments: (arguments . (object) @options)) + `, +}; + +/** + * Receivers admitted as wrapped HTTP clients without axios.create proof. + * Spelling-only: the last identifier in `obj.text` (`this.$http` → `$http`). + * Keep this set small — every extra name is a false-positive surface. + */ +const WRAPPED_REQUEST_RECEIVERS = new Set(['httpClient', '$http']); + interface NodePatternBundle { express: CompiledPatterns>; fetchNoOptions: CompiledPatterns>; @@ -160,6 +192,7 @@ interface NodePatternBundle { jqueryShorthand: CompiledPatterns>; jqueryAjax: CompiledPatterns>; axiosObject: CompiledPatterns>; + requestObject: CompiledPatterns>; } function compileBundle(language: unknown, name: string): NodePatternBundle { @@ -177,6 +210,7 @@ function compileBundle(language: unknown, name: string): NodePatternBundle { jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'), jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'), axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'), + requestObject: mk(REQUEST_OBJECT_SPEC, 'request-object'), }; } @@ -190,6 +224,7 @@ const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http'); * of `keyNames`. Returns null when no matching pair is present or the * value is not a string literal. Used by the jQuery ajax / axios object * consumers to resolve `url` / `method` / `type` keys in any order. + * Keys use shared `propertyName` so quoted `"method"` matches `method`. */ function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string[]): string | null { for (let i = 0; i < objectNode.namedChildCount; i++) { @@ -198,7 +233,8 @@ function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string const keyNode = pair.childForFieldName('key'); const valueNode = pair.childForFieldName('value'); if (!keyNode || !valueNode) continue; - if (!keyNames.includes(keyNode.text)) continue; + const key = propertyName(keyNode); + if (key === null || !keyNames.includes(key)) continue; if (valueNode.type !== 'string' && valueNode.type !== 'template_string') continue; const lit = unquoteLiteral(valueNode.text); if (lit !== null) return lit; @@ -206,6 +242,80 @@ function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string return null; } +/** + * Verb for wrapped `X.request({ url, method|type })`. Absent key → GET + * (same default as fetch-without-options / jQuery ajax). Present but not a + * string/template, supplied only via object spread, or later overwritten by + * a duplicate key / spread → `*` so matching can still link without pinning GET. + * Later properties win, matching JavaScript object-literal evaluation. + */ +function readRequestMethod( + objectNode: Parser.SyntaxNode, + keyNames: readonly string[] = ['method', 'type'], +): string { + type Verb = { kind: 'absent' } | { kind: 'literal'; value: string } | { kind: 'unknown' }; + let last: Verb = { kind: 'absent' }; + for (let i = 0; i < objectNode.namedChildCount; i++) { + const child = objectNode.namedChild(i); + if (!child) continue; + if (child.type === 'spread_element') { + last = { kind: 'unknown' }; + continue; + } + if ( + child.type === 'shorthand_property_identifier' || + child.type === 'shorthand_property_identifier_pattern' + ) { + if (keyNames.includes(child.text)) last = { kind: 'unknown' }; + continue; + } + if (child.type !== 'pair') continue; + const keyNode = child.childForFieldName('key'); + const valueNode = child.childForFieldName('value'); + if (!keyNode) continue; + const key = propertyName(keyNode); + if (key === null || !keyNames.includes(key)) continue; + if (!valueNode || (valueNode.type !== 'string' && valueNode.type !== 'template_string')) { + last = { kind: 'unknown' }; + continue; + } + const lit = unquoteLiteral(valueNode.text); + if (lit === null || lit.includes('${')) { + last = { kind: 'unknown' }; + continue; + } + last = { kind: 'literal', value: lit }; + } + if (last.kind === 'literal') return last.value.toUpperCase(); + if (last.kind === 'unknown') return '*'; + return 'GET'; +} + +function wrappedRequestReceiverName(receiver: string): string { + const parts = receiver.split('.'); + return parts[parts.length - 1] ?? receiver; +} + +/** Axios module / axios.create instance, or a registered wrapper identifier. */ +function isAdmittedWrappedRequestReceiver( + receiver: string, + fileKey: string | undefined, + facts: JsRepoFacts | null, +): boolean { + if (WRAPPED_REQUEST_RECEIVERS.has(wrappedRequestReceiverName(receiver))) return true; + try { + const isModule = + facts === null || fileKey === undefined + ? receiver === 'axios' + : isAxiosNamespace(fileKey, receiver, facts); + if (isModule) return true; + if (!facts || fileKey === undefined) return false; + return isHttpClientRef(fileKey, receiver, facts); + } catch { + return false; + } +} + /** * Map each named import's LOCAL binding to its DECLARED export name and source * module, by walking the file's `import { x as y } from 'm'` statements. Lets @@ -390,9 +500,10 @@ function resolveFactsFor( * * `/{param}` matches every one-segment provider route in the group, and * `matching.exclude_links_param_only_paths` defaults to `false`. A path whose - * leading term is an unresolved placeholder is refused for the same reason — - * nothing pins where it starts. (`resolveJsPathExpression` already refuses those - * it folded itself; this also covers the literal fallback below.) + * leading term is an unresolved placeholder is refused unless the next + * character is `/` — that is the gateway-prefix shape + * `` `${serviceClient}/api/v1/x` `` that `stripLeadingTemplatePrefix` keeps. + * Bare `{param}` and `{param}api/x` stay rejected: nothing pins a route. */ function looksLikeHttpPath(path: string): boolean { if (path === '') return false; @@ -404,7 +515,7 @@ function looksLikeHttpPath(path: string): boolean { // unresolved term happened to contain a space. const shape = path.replace(/\$\{[^}]+\}/g, '{param}'); if (/\s/.test(shape)) return false; - if (shape.startsWith('{param}')) return false; + if (shape.startsWith('{param}') && !shape.startsWith('{param}/')) return false; // An all-digit string is a path only when it is written as one. A leading // slash is that evidence: `client.get('/123')` is a route whose segment the // consumer normalizer reads as `{param}`, while a bare `"5000"` folded out of @@ -672,8 +783,7 @@ function scanBundle( if (!optionsNode) continue; const path = readStringProp(optionsNode, ['url']); if (path === null) continue; - const rawMethod = readStringProp(optionsNode, ['method']); - const method = (rawMethod ?? 'GET').toUpperCase(); + const method = readRequestMethod(optionsNode, ['method']); out.push({ role: 'consumer', framework: 'axios', @@ -685,6 +795,37 @@ function scanBundle( }); } + // Consumer: wrapped client `X.request({ url, method })` — the shared + // enterprise axios-instance shape (`httpClient.request` from + // win-request and friends). Emit the raw url (templates intact) so + // shared `normalizeConsumerPath` can strip a leading `${…}` gateway + // prefix and fold mid/tail interpolations to `{param}`. A plugin-side + // longest-slash-segment reducer would truncate those mid-templates + // before the shared normalizer ever saw them. This scan drops only + // static relative urls (no `${`, no leading `/`, not `https?://`). + // That filter is request-wrapper-specific: fetch/axios member forms + // already admit absolute urls and leave host stripping to + // `normalizeConsumerPath`. + for (const match of runCompiledPatterns(bundle.requestObject, tree)) { + const optionsNode = match.captures.options; + const objNode = match.captures.obj; + if (!optionsNode || !objNode) continue; + if (!isAdmittedWrappedRequestReceiver(objNode.text, fileKey, facts)) continue; + const rawUrl = readStringProp(optionsNode, ['url']); + if (rawUrl === null) continue; + const url = rawUrl.trim(); + if (!url.includes('${') && !url.startsWith('/') && !/^https?:\/\//i.test(url)) continue; + out.push({ + role: 'consumer', + framework: 'request', + method: readRequestMethod(optionsNode), + path: url, + name: null, + line: optionsNode.startPosition.row + 1, + confidence: 0.65, + }); + } + for (const route of scanDataRouteTables(tree)) { const imported = route.handlerLocalName === undefined ? undefined : importMap.get(route.handlerLocalName); diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 5def6a9a9..d116bc046 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -292,13 +292,57 @@ export function normalizeHttpPath(p: string): string { } /** - * Consumer-side normalization is more aggressive: - * - template literals (`${x}`) → `{param}` - * - strip protocol + host if the URL is absolute - * - numeric segments → `{param}` (so `/api/orders/42` → `/api/orders/{param}`) + * Strip LEADING template interpolations from a consumer url as gateway/host + * bindings — the enterprise wrapper shape `` `${serviceClient}/api/v1/x` `` + * where `${serviceClient}` selects the gateway service, not a route segment. + * This is consumer-path framework semantics (mirrors how an absolute + * `https://host/path` url keeps only its path), so it lives here rather than + * in any one language plugin: + * - `` `${c}/api/x` `` → `/api/x` (clean prefix; `${c}${d}/api/x` → `/api/x`) + * - `` `${c}/api/x/${id}` ``→ `/api/x/${id}` (mid/tail interpolations are left for the `{param}` pass) + * Returns null when the stripped remainder is not a single-slash path: + * - remainder without `/` — relative fragment (`${c}api/x`) or scheme/host + * (`${scheme}://${host}/api/x`); whether it is a path depends on + * unverifiable runtime state, so it is dropped rather than guessed at; + * - remainder starting with `//` — protocol-relative (`${proto}//host/api/x`); + * keeping it would later collapse to `/host/api/x`. + * + * The `?` in a query string cannot leak into the brace matching: `${...}` + * spans are matched by braces here (before any `{param}` replacement), and + * `normalizeHttpPath` splits on `?` only after the whole `${...}` span — + * including any `?` inside it — has been collapsed to `{param}`. So + * `` `${c}/api/x?id=${id}` `` reduces to `/api/x` on both orderings. */ -function normalizeConsumerPath(url: string): string { - const templated = url.replace(/\$\{[^}]+\}/g, '{param}').trim(); +function stripLeadingTemplatePrefix(url: string): string | null { + if (!url.startsWith('${')) return url; + const rest = url.replace(/^(?:\$\{[^}]*\})+/, ''); + return rest.startsWith('/') && !rest.startsWith('//') ? rest : null; +} + +/** + * Placeholder substituted for `${...}` before WHATWG `URL` parsing so the + * parser cannot percent-encode our own `{param}` markers. A genuine encoded + * segment like `%7Bfoo%7D` then survives as a literal, instead of being + * rewritten into braces and folded into `{param}`. + * + * Private-use U+E000 cannot appear in a real URL path, so a literal + * `__gitnexus_http_param__` segment is not rewritten into `{param}`. + */ +const CONSUMER_PARAM_SENTINEL = '\uE000'; +const CONSUMER_PARAM_SENTINEL_ENC = '%ee%80%80'; + +function restoreConsumerParamSentinel(pathOnly: string): string { + return pathOnly + .split(CONSUMER_PARAM_SENTINEL) + .join('{param}') + .replace(new RegExp(CONSUMER_PARAM_SENTINEL_ENC, 'gi'), '{param}'); +} + +/** Canonicalize a consumer URL after `stripLeadingTemplatePrefix`. */ +function normalizeConsumerPath(url: string): string | null { + const stripped = stripLeadingTemplatePrefix(url.trim()); + if (stripped === null) return null; + const templated = stripped.replace(/\$\{[^}]+\}/g, CONSUMER_PARAM_SENTINEL).trim(); let pathOnly = templated; if (/^https?:\/\//i.test(templated)) { try { @@ -307,6 +351,7 @@ function normalizeConsumerPath(url: string): string { pathOnly = templated.replace(/^https?:\/\/[^/]+/i, ''); } } + pathOnly = restoreConsumerParamSentinel(pathOnly); const normalized = normalizeHttpPath(pathOnly || '/'); const segments = normalized .split('/') @@ -999,6 +1044,12 @@ export class HttpRouteExtractor implements ContractExtractor { for (const d of detections) { if (d.role !== 'consumer') continue; const pathNorm = normalizeConsumerPath(d.path); + // A consumer url that cannot be reduced to a routable path (e.g. a + // leading template binding that is neither a clean prefix nor a + // remainder that starts with `/`) is dropped here rather than emitted + // as a never-matching contract — same treatment the plugins give + // static relative urls at scan time. + if (pathNorm === null) continue; // Resolve the function CONTAINING the fetch/axios call so the consumer // contract carries a real symbolUid (was always '' — the gap that left // cross-repo trace/impact unable to traverse HTTP links). diff --git a/gitnexus/test/unit/group/http-route-extractor.test.ts b/gitnexus/test/unit/group/http-route-extractor.test.ts index 68e7d8669..b59ca8a3b 100644 --- a/gitnexus/test/unit/group/http-route-extractor.test.ts +++ b/gitnexus/test/unit/group/http-route-extractor.test.ts @@ -2764,6 +2764,12 @@ export function updateUser(id: string, data: unknown) { export function listDefaults() { return axios({ url: '/api/defaults' }); } +export function ignoreJqueryTypeKey() { + return axios({ url: '/api/typed', type: 'POST' }); +} +export function quotedAxiosMethod() { + return axios({ url: '/api/quoted-ax', "method": 'DELETE' }); +} `, ); @@ -2773,6 +2779,111 @@ export function listDefaults() { expect(consumers.find((c) => c.contractId === 'http::POST::/api/orders')).toBeDefined(); expect(consumers.find((c) => c.contractId === 'http::PUT::/api/users/{param}')).toBeDefined(); expect(consumers.find((c) => c.contractId === 'http::GET::/api/defaults')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::POST::/api/typed')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/typed')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::DELETE::/api/quoted-ax')).toBeDefined(); + }); + + it('extracts wrapped X.request({ url, method }) with shared prefix-strip and * verbs', async () => { + const dir = path.join(tmpDir, 'wrapped-request'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src/client.ts'), + ` +import axios from 'axios'; + +export function listOrders(httpClient, serviceClient, tenant, verb) { + return httpClient.request({ url: \`\${serviceClient}/api/v1/orders\`, method: 'post' }); +} +export function getTenantOrder(httpClient, client, tenant) { + return httpClient.request({ url: \`\${client}/api/\${tenant}/orders\`, method: 'GET' }); +} +export function rootPing(httpClient, client) { + return httpClient.request({ url: \`\${client}/\`, method: 'GET' }); +} +export function dynamicVerb(httpClient) { + return httpClient.request({ url: '/api/orders', method: verb }); +} +export function missingMethod(httpClient) { + return httpClient.request({ url: '/api/defaults' }); +} +export function dropHostTemplate(httpClient, scheme, host) { + return httpClient.request({ url: \`\${scheme}://\${host}/api/x\`, method: 'GET' }); +} +export function absTemplateParam(httpClient, id) { + return httpClient.request({ url: \`https://host/api/\${id}\`, method: 'GET' }); +} +export function quotedMethod(httpClient) { + return httpClient.request({ url: '/api/quoted', "method": 'PATCH' }); +} +export function spreadMethod(httpClient, config) { + return httpClient.request({ url: '/api/spread', ...config }); +} +export function staticAbsolute(httpClient) { + return httpClient.request({ url: 'https://host/api/static', method: 'GET' }); +} +export function protocolRelative(httpClient, proto) { + return httpClient.request({ url: \`\${proto}//host/api/proto\`, method: 'GET' }); +} +export async function fetchGateway(gateway) { + return fetch(\`\${gateway}/api/users\`); +} +export function axiosGateway(gateway) { + return axios.get(\`\${gateway}/api/users\`); +} +export async function encodedBraceLiteral() { + return fetch('https://host/api/%7Bfoo%7D'); +} +export async function literalSentinelSegment() { + return fetch('https://host/api/__gitnexus_http_param__'); +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers.find((c) => c.contractId === 'http::POST::/api/v1/orders')).toBeDefined(); + expect( + consumers.find((c) => c.contractId === 'http::GET::/api/{param}/orders'), + ).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::*::/api/orders')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/defaults')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/x')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/orders')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/{param}')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::PATCH::/api/quoted')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::*::/api/spread')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/static')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/proto')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/host/api/proto')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/users')).toBeDefined(); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/%7bfoo%7d')).toBeDefined(); + expect( + consumers.find((c) => c.contractId === 'http::GET::/api/__gitnexus_http_param__'), + ).toBeDefined(); + }); + + it('does not mint HTTP consumers for ungated .request({ url }) helpers', async () => { + const dir = path.join(tmpDir, 'wrapped-request-negative'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src/misc.ts'), + ` +export function e2e(cy) { + return cy.request({ url: '/api/v1/orders', method: 'GET' }); +} +export function enqueue(queue) { + return queue.request({ url: '/admin', method: 'DELETE' }); +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + expect(consumers.find((c) => c.contractId === 'http::GET::/api/v1/orders')).toBeUndefined(); + expect(consumers.find((c) => c.contractId === 'http::DELETE::/admin')).toBeUndefined(); }); it('does not emit consumers for unrelated object-literal calls (negative control)', async () => { diff --git a/gitnexus/test/unit/group/js-http-consumer-resolution.test.ts b/gitnexus/test/unit/group/js-http-consumer-resolution.test.ts index 8957825dc..c39dda791 100644 --- a/gitnexus/test/unit/group/js-http-consumer-resolution.test.ts +++ b/gitnexus/test/unit/group/js-http-consumer-resolution.test.ts @@ -843,3 +843,198 @@ describe('resolveJsImport', () => { ); }); }); + +describe('wrapped X.request({ url, method }) detections', () => { + it('keeps the raw template so mid-path interpolations survive to the normalizer', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse( + 'httpClient.request({ url: `${client}/api/${tenant}/orders`, method: "GET" });', + ), + ), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.framework).toBe('request'); + expect(detections[0]?.method).toBe('GET'); + expect(detections[0]?.path).toBe('${client}/api/${tenant}/orders'); + }); + + it('emits * when method is present but not a literal', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse('httpClient.request({ url: "/api/orders", method: verb });'), + ), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.method).toBe('*'); + }); + + it('uses the last duplicate method key, matching JS evaluation', () => { + const dynamicLast = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: '/api/orders', method: 'GET', method: verb });"), + ), + ); + expect(dynamicLast[0]?.method).toBe('*'); + const literalLast = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: '/api/orders', method: verb, method: 'POST' });"), + ), + ); + expect(literalLast[0]?.method).toBe('POST'); + }); + + it('defaults to GET only when method/type is absent', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan(jsParser.parse('httpClient.request({ url: "/api/orders" });')), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.method).toBe('GET'); + }); + + it('drops a static relative url at scan time', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: 'api/orders', method: 'GET' });"), + ), + ); + expect(detections).toHaveLength(0); + }); + + it('does not treat cy.request or queue.request as HTTP consumers', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse(` +cy.request({ url: '/api/v1/orders', method: 'GET' }); +queue.request({ url: '/admin', method: 'DELETE' }); +`), + ), + ); + expect(detections).toHaveLength(0); + }); + + it('ignores a config object that is not the first request argument', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request('/actual', { url: '/metadata', method: 'GET' });"), + ), + ); + expect(detections).toHaveLength(0); + }); + + it('admits $http by spelling but not a bare api without axios proof', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse(` +$http.request({ url: '/api/orders', method: 'GET' }); +api.request({ url: '/api/users', method: 'POST' }); +`), + ), + ); + expect(detections).toEqual([ + expect.objectContaining({ method: 'GET', path: '/api/orders', framework: 'request' }), + ]); + }); + + it('reads quoted method/type keys and type: as the verb', () => { + const quoted = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse('httpClient.request({ url: "/api/orders", "method": "POST" });'), + ), + ); + expect(quoted[0]?.method).toBe('POST'); + const typed = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: '/api/items', type: 'PUT' });"), + ), + ); + expect(typed[0]?.method).toBe('PUT'); + }); + + it('emits * when method may arrive via object spread', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse('httpClient.request({ url: "/api/orders", ...config });'), + ), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.method).toBe('*'); + }); + + it('keeps static absolute wrapped-request urls for host stripping', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: 'https://host/api/x', method: 'GET' });"), + ), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.path).toBe('https://host/api/x'); + }); + + it('admits axios.create instances calling .request', () => { + const detections = consumers( + scanRepo( + { + 'src/lib/client.ts': ` + import axios from 'axios'; + export const api = axios.create({ baseURL: '/' }); + `, + 'src/api/orders.ts': ` + import { api } from '../lib/client'; + export const create = () => api.request({ url: '/api/orders', method: 'POST' }); + `, + }, + 'src/api/orders.ts', + ), + ); + expect(detections).toContainEqual( + expect.objectContaining({ role: 'consumer', method: 'POST', path: '/api/orders' }), + ); + }); + + it('admits member-verb calls with a gateway-prefixed template', () => { + const detections = consumers( + scanRepo( + { + 'src/lib/client.ts': ` + import axios from 'axios'; + export default axios.create({ baseURL: '/' }); + `, + 'src/api/users.ts': ` + import api from '../lib/client'; + export const list = (gateway: string) => api.get(\`\${gateway}/api/v1/users\`); + export const bare = (id: string) => api.get(\`\${id}\`); + export const glue = (c: string) => api.get(\`\${c}api/x\`); + `, + }, + 'src/api/users.ts', + ), + ); + expect(detections.map((d) => d.path)).toEqual(['${gateway}/api/v1/users']); + }); + + it('trims whitespace-prefixed absolute paths at scan time', () => { + const detections = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: ' /api/orders', method: 'GET' });"), + ), + ); + expect(detections).toHaveLength(1); + expect(detections[0]?.path).toBe('/api/orders'); + }); + + it('emits * for interpolated method templates and shorthand method keys', () => { + const interpolated = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse('httpClient.request({ url: "/api/orders", method: `${verb}` });'), + ), + ); + expect(interpolated[0]?.method).toBe('*'); + const shorthand = consumers( + JAVASCRIPT_HTTP_PLUGIN.scan( + jsParser.parse("httpClient.request({ url: '/api/orders', method });"), + ), + ); + expect(shorthand[0]?.method).toBe('*'); + }); +}); From 131d9f93fd3e0ae970ac3bb69e0d2770003918fe Mon Sep 17 00:00:00 2001 From: ChunxueLi <54129170+ChunxueLi@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:01:59 +0800 Subject: [PATCH 04/21] feat(route): resolve vendor-derived Spring mapping annotations by suffix (#2883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): resolve vendor-derived Spring mapping annotations by suffix Frameworks commonly wrap Spring's built-in annotations with company-specific variants (e.g. Winning Health's @WinPostMapping wraps @PostMapping). The annotation definition lives in a binary JAR — not in source — so the meta-annotation cannot be read statically. Add resolveSpringAnnotationAlias(): resolves custom annotations by naming suffix (WinPostMapping → PostMapping → POST). This matches the universal Java convention of naming derived annotations with the base name as a suffix. Works for any vendor prefix, not just one company. The fix is in springAnnotationHttpMethods() (spring-shared.ts), which both the ingestion extractor (spring.ts) and the group extractor (java.ts) call. A single-function change propagates to both layers automatically. Zero configuration: no .gitnexusrc, no annotation allowlist. If an annotation name ends with a known Spring mapping suffix, it inherits that annotation's HTTP semantics. False-positive risk is negligible. Tests: 16 new unit tests covering resolveSpringAnnotationAlias directly, springAnnotationHttpMethods with aliased annotations, end-to-end extractSpringRoutes with vendor annotations, and ingestion/group parity. Existing route tests (260) continue to pass. * fix(route): address review findings — class-level aliases, registered prefixes P1: class-level @WinRequestMapping now gets the same prefix/constraint semantics as @RequestMapping — all five class-level exact-match sites (spring.ts phase-1 collect, typeRequestMethods, typeClassPrefixes; group http-patterns java.ts typeRequestMethods + type-level branch) route through the new shared isClassLevelMappingAnnotation predicate, and the hard-coded 'RequestMapping' argument in springAnnotationHttpMethods calls is replaced with the actual annotation name so alias resolution applies. P2: suffix-only alias matching accepted unrelated annotations (@AuditPostMapping emitted a phantom POST /audit). Alias resolution now requires a REGISTERED vendor prefix — 'Win' by default, extendable via GITNEXUS_SPRING_VENDOR_PREFIXES=Win,Acme without a rebuild. Tests: negative e2e for the unregistered-suffix phantom route, vendor class-prefix parity e2e, predicate unit matrix, env-registration test. * style: prettier * chore: drop accidental gitnexus-shared/dist worktree symlink from prettier commit Co-authored-by: Cursor * Address PR review feedback (#2883) Wire vendor Spring mapping aliases into Kotlin ingestion and group extraction, restore GITNEXUS_SPRING_VENDOR_PREFIXES after the env test, and stamp spring.route-bindings so existing indexes rebuild. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command * fix(route): honor Kotlin vendor aliases and prefix freshness Parse Kotlin RequestMapping method arrays in the shared Spring helper, bump spring.route-bindings, and rebuild when registered vendor prefixes change. Co-authored-by: Cursor --------- Co-authored-by: ChunxueLi Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- .../src/core/analysis-feature-registry.ts | 26 ++ .../group/extractors/http-patterns/java.ts | 13 +- .../group/extractors/http-patterns/kotlin.ts | 128 ++++++-- .../frameworks/spring/analysis-features.ts | 10 + .../frameworks/spring/vendor-prefixes.ts | 27 ++ .../route-extractors/kotlin-spring.ts | 24 +- .../route-extractors/spring-shared.ts | 94 +++++- .../core/ingestion/route-extractors/spring.ts | 17 +- gitnexus/src/core/run-analyze.ts | 41 ++- gitnexus/src/storage/repo-meta.ts | 5 + gitnexus/test/unit/analysis-features.test.ts | 38 +-- .../unit/incremental-orchestration.test.ts | 37 +++ .../kotlin-spring-route-ingestion.test.ts | 157 ++++++++- .../spring-vendor-annotation-alias.test.ts | 297 ++++++++++++++++++ 14 files changed, 786 insertions(+), 128 deletions(-) create mode 100644 gitnexus/src/core/analysis-feature-registry.ts create mode 100644 gitnexus/src/core/ingestion/frameworks/spring/vendor-prefixes.ts create mode 100644 gitnexus/test/unit/spring-vendor-annotation-alias.test.ts diff --git a/gitnexus/src/core/analysis-feature-registry.ts b/gitnexus/src/core/analysis-feature-registry.ts new file mode 100644 index 000000000..474c2157d --- /dev/null +++ b/gitnexus/src/core/analysis-feature-registry.ts @@ -0,0 +1,26 @@ +import { CLASS_FRAMEWORK_ANNOTATIONS_FEATURE } from './analysis-features.js'; +import { + SPRING_AOP_FEATURE, + SPRING_BEAN_INVENTORY_FEATURE, + SPRING_CONDITIONALS_FEATURE, + SPRING_NON_HTTP_HANDLERS_FEATURE, + SPRING_ROUTE_BINDINGS_FEATURE, +} from './ingestion/frameworks/spring/analysis-features.js'; +import { + JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, + JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, + SPRING_CONFIG_BINDINGS_FEATURE, +} from './ingestion/languages/java/analysis-features.js'; + +/** Production registry of independently versioned analysis capabilities. */ +export const ANALYSIS_FEATURES = [ + CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, + SPRING_AOP_FEATURE, + SPRING_BEAN_INVENTORY_FEATURE, + SPRING_CONDITIONALS_FEATURE, + SPRING_NON_HTTP_HANDLERS_FEATURE, + SPRING_ROUTE_BINDINGS_FEATURE, + SPRING_CONFIG_BINDINGS_FEATURE, + JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, + JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, +] as const; diff --git a/gitnexus/src/core/group/extractors/http-patterns/java.ts b/gitnexus/src/core/group/extractors/http-patterns/java.ts index 8d216c50a..8b8897a91 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/java.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/java.ts @@ -11,6 +11,7 @@ import { intersectSpringHttpMethods, isRouteMemberKey, findEnclosingClass, + isClassLevelMappingAnnotation, joinPath, type SharedSpringType, } from '../../../ingestion/route-extractors/spring-shared.js'; @@ -467,13 +468,15 @@ function annotationHasRouteMember(annotation: Parser.SyntaxNode): boolean { } function typeRequestMethods(typeNode: Parser.SyntaxNode): readonly string[] { - const mappings = declarationAnnotations(typeNode).filter( - (annotation) => - simpleName(annotation.childForFieldName('name')?.text ?? '') === 'RequestMapping', + const mappings = declarationAnnotations(typeNode).filter((annotation) => + isClassLevelMappingAnnotation(simpleName(annotation.childForFieldName('name')?.text ?? '')), ); if (mappings.length === 0) return ['*']; if (mappings.length !== 1) return []; - return springAnnotationHttpMethods('RequestMapping', mappings[0].text); + return springAnnotationHttpMethods( + simpleName(mappings[0].childForFieldName('name')?.text ?? 'RequestMapping'), + mappings[0].text, + ); } function hasAnnotation(node: Parser.SyntaxNode, names: string | readonly string[]): boolean { @@ -675,7 +678,7 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { // Type-level (class or interface): a Spring `@RequestMapping` URL prefix, or // — on an interface — an OpenFeign `@FeignClient(path = "...")` prefix. - if (ann === 'RequestMapping') { + if (isClassLevelMappingAnnotation(ann)) { if (!isRouteMemberKey(keyNode)) continue; if (!valueNode) { // Constant-valued class prefix — see `typesWithUnfoldablePrefix`. diff --git a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts index ec695a9b6..b00a3e400 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts @@ -13,9 +13,11 @@ import type { HttpScanInput, } from './types.js'; import { - METHOD_ANNOTATION_TO_HTTP, findEnclosingClass, + intersectSpringHttpMethods, + isClassLevelMappingAnnotation, joinPath, + springAnnotationHttpMethods, type SharedSpringType, } from '../../../ingestion/route-extractors/spring-shared.js'; import { @@ -448,6 +450,13 @@ function inferKotlinOkHttpMethod(urlCall: Parser.SyntaxNode): string | null { return name === null ? 'GET' : name.toUpperCase(); } +function enclosingAnnotationText(node: Parser.SyntaxNode): string { + for (let current: Parser.SyntaxNode | null = node; current; current = current.parent) { + if (current.type === 'annotation') return current.text; + } + return node.text; +} + /** * Build the plugin only if the Kotlin grammar is available. Compiling * the queries against a null grammar would throw at module load time @@ -485,7 +494,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (user_type (type_identifier) @ann (#match? @ann "RequestMapping$")) (value_arguments (value_argument . [(string_literal) @prefix (collection_literal (string_literal) @prefix)]))))) (type_identifier) @cls) @class @@ -498,7 +507,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (user_type (type_identifier) @ann (#match? @ann "RequestMapping$")) (value_arguments (value_argument (simple_identifier) @key (#match? @key "^(path|value)$") @@ -513,7 +522,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (user_type (type_identifier) @ann (#match? @ann "RequestMapping$")) (value_arguments (value_argument . ${arrayOfArg('@prefix')}))))) (type_identifier) @cls) @class @@ -526,7 +535,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (user_type (type_identifier) @ann (#match? @ann "RequestMapping$")) (value_arguments (value_argument (simple_identifier) @key (#match? @key "^(path|value)$") @@ -552,7 +561,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (user_type (type_identifier) @ann (#match? @ann "(Request|Get|Post|Put|Delete|Patch)Mapping$")) (value_arguments (value_argument . [(string_literal) @path (collection_literal (string_literal) @path)]))))) (simple_identifier) @method_name) @method @@ -565,7 +574,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (user_type (type_identifier) @ann (#match? @ann "(Request|Get|Post|Put|Delete|Patch)Mapping$")) (value_arguments (value_argument (simple_identifier) @key (#match? @key "^(path|value)$") @@ -580,7 +589,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (user_type (type_identifier) @ann (#match? @ann "(Request|Get|Post|Put|Delete|Patch)Mapping$")) (value_arguments (value_argument . ${arrayOfArg('@path')}))))) (simple_identifier) @method_name) @method @@ -593,7 +602,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (user_type (type_identifier) @ann (#match? @ann "(Request|Get|Post|Put|Delete|Patch)Mapping$")) (value_arguments (value_argument (simple_identifier) @key (#match? @key "^(path|value)$") @@ -629,7 +638,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (user_type (type_identifier) @ann (#match? @ann "RequestMapping$")) (value_arguments (value_argument) @arg)))) (type_identifier) @cls) @class `, @@ -648,7 +657,7 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { (modifiers (annotation (constructor_invocation - (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (user_type (type_identifier) @ann (#match? @ann "(Request|Get|Post|Put|Delete|Patch)Mapping$")) (value_arguments (value_argument) @arg)))) (simple_identifier) @method_name) @method `, @@ -713,7 +722,9 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { for (const match of runCompiledPatterns(SPRING_CONST_CLASS_PREFIX_PATTERNS, tree)) { const argNode = match.captures.arg; const classNode = match.captures.class; + const annNode = match.captures.ann; if (!argNode || !classNode) continue; + if (annNode && !isClassLevelMappingAnnotation(annNode.text)) continue; if ((resolvedPrefixes.get(classNode.id) ?? []).length > 0) continue; const expr = kotlinRouteArgumentExpression(argNode); if (!expr || classifyPathArgument(expr) !== 'unresolvable') continue; @@ -1218,13 +1229,36 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { const kotlinFunctionName = (fn: Parser.SyntaxNode): string | null => fn.namedChildren.find((c) => c.type === 'simple_identifier')?.text ?? null; + const kotlinTypeRequestMethods = (typeNode: Parser.SyntaxNode): readonly string[] => { + const modifiers = typeNode.namedChildren.find((child) => child.type === 'modifiers'); + const mappings = (modifiers?.namedChildren ?? []).filter((annotation) => { + if (annotation.type !== 'annotation') return false; + return isClassLevelMappingAnnotation(kotlinAnnotationName(annotation) ?? ''); + }); + if (mappings.length === 0) return ['*']; + if (mappings.length !== 1) return []; + const mapping = mappings[0]; + const mappingName = kotlinAnnotationName(mapping); + if (!mappingName) return []; + return springAnnotationHttpMethods(mappingName, mapping.text); + }; + + const kotlinClassHttpMethodsById = (tree: Parser.Tree) => + new Map( + tree.rootNode + .descendantsOfType('class_declaration') + .map((typeNode) => [typeNode.id, kotlinTypeRequestMethods(typeNode)] as const), + ); + const collectKotlinSpringTypes = (filePath: string, tree: Parser.Tree): SharedSpringType[] => { // Class-level @RequestMapping prefixes (reuse the provider class-prefix query). const prefixByClassId = new Map(); for (const match of runCompiledPatterns(SPRING_CLASS_PREFIX_PATTERNS, tree)) { const prefixNode = match.captures.prefix; const classNode = match.captures.class; + const annNode = match.captures.ann; if (!prefixNode || !classNode) continue; + if (annNode && !isClassLevelMappingAnnotation(annNode.text)) continue; // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — unquoting // its raw text would carry the source spelling into the shared type view // as a served prefix. Refusing it here is also what lets the unfoldable @@ -1242,22 +1276,32 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { // noise into the shared type view, so it is left out — the same skip floor // `java.ts`'s `collectSpringTypes` keeps. const routesByMethodId = new Map>(); + const classHttpMethodsById = kotlinClassHttpMethodsById(tree); const unfoldablePrefixClassIds = collectUnfoldablePrefixClassIds(tree, prefixByClassId); for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; const methodNode = match.captures.method; if (!annNode || !pathNode || !methodNode) continue; - const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; - if (!httpMethod) continue; + const httpMethods = springAnnotationHttpMethods( + annNode.text, + enclosingAnnotationText(annNode), + ); + if (httpMethods.length === 0) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; // A constant class prefix leaves no single prefix string for the // inheritance view to carry, so this route would be published unprefixed. const owner = findEnclosingClass(methodNode); if (owner && unfoldablePrefixClassIds.has(owner.id)) continue; + const constrainedMethods = intersectSpringHttpMethods( + owner ? (classHttpMethodsById.get(owner.id) ?? ['*']) : ['*'], + httpMethods, + ); const arr = routesByMethodId.get(methodNode.id) ?? []; - arr.push({ method: httpMethod, path: rawPath }); + for (const httpMethod of constrainedMethods) { + arr.push({ method: httpMethod, path: rawPath }); + } routesByMethodId.set(methodNode.id, arr); } @@ -1390,7 +1434,9 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { for (const match of runCompiledPatterns(SPRING_CLASS_PREFIX_PATTERNS, tree)) { const prefixNode = match.captures.prefix; const classNode = match.captures.class; + const annNode = match.captures.ann; if (!prefixNode || !classNode) continue; + if (annNode && !isClassLevelMappingAnnotation(annNode.text)) continue; // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — see // `isPlainStringLiteral`. Refusing it here also lets the unfoldable // analysis below mark such a class, since that skips classes whose @@ -1438,29 +1484,38 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { nameNode: Parser.SyntaxNode | undefined; methodNode: Parser.SyntaxNode; }> = []; + const classHttpMethodsById = kotlinClassHttpMethodsById(tree); for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; const methodNode = match.captures.method; if (!annNode || !pathNode || !methodNode) continue; - const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; - if (!httpMethod) continue; + const httpMethods = springAnnotationHttpMethods( + annNode.text, + enclosingAnnotationText(annNode), + ); + if (httpMethods.length === 0) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; - methodRoutes.push({ - httpMethod, - rawPath, - nameNode: match.captures.method_name, - methodNode, - }); + for (const httpMethod of httpMethods) { + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } } for (const match of runCompiledPatterns(SPRING_CONST_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const argNode = match.captures.arg; const methodNode = match.captures.method; if (!annNode || !argNode || !methodNode) continue; - const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; - if (!httpMethod) continue; + const httpMethods = springAnnotationHttpMethods( + annNode.text, + enclosingAnnotationText(annNode), + ); + if (httpMethods.length === 0) continue; const expr = kotlinRouteArgumentExpression(argNode); if (!expr || !FOLDABLE_PATH_EXPRESSIONS.has(expr.type)) continue; // No repo context (context-less fallback scanning) means no constant map @@ -1481,15 +1536,26 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { index, ); if (rawPath === null) continue; - methodRoutes.push({ - httpMethod, - rawPath, - nameNode: match.captures.method_name, - methodNode, - }); + for (const httpMethod of httpMethods) { + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } } - for (const { httpMethod, rawPath, nameNode, methodNode } of methodRoutes) { + const constrainedMethodRoutes = methodRoutes.flatMap((route) => { + const owner = findEnclosingClass(route.methodNode); + const classMethods = owner ? (classHttpMethodsById.get(owner.id) ?? ['*']) : ['*']; + return intersectSpringHttpMethods(classMethods, [route.httpMethod]).map((httpMethod) => ({ + ...route, + httpMethod, + })); + }); + + for (const { httpMethod, rawPath, nameNode, methodNode } of constrainedMethodRoutes) { const enclosingClass = findEnclosingClass(methodNode); // A @(Get|...)Mapping inside a @FeignClient interface is an OpenFeign // consumer (a remote call), not a route this service serves. diff --git a/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts b/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts index ca79ed745..722c53249 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts @@ -50,3 +50,13 @@ export const SPRING_NON_HTTP_HANDLERS_FEATURE: AnalysisFeatureDescriptor = { version: 1, appliesTo: (filePaths) => filePaths.some(isJvmSourceFile), }; + +/** + * Route/handler binding extraction, including vendor `@Win*Mapping` aliases. + * Existing indexes keep a stale Route set until this version is stamped. + */ +export const SPRING_ROUTE_BINDINGS_FEATURE: AnalysisFeatureDescriptor = { + id: 'spring.route-bindings', + version: 2, + appliesTo: (filePaths) => filePaths.some(isJvmSourceFile), +}; diff --git a/gitnexus/src/core/ingestion/frameworks/spring/vendor-prefixes.ts b/gitnexus/src/core/ingestion/frameworks/spring/vendor-prefixes.ts new file mode 100644 index 000000000..8134104b4 --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/vendor-prefixes.ts @@ -0,0 +1,27 @@ +const DEFAULT_SPRING_VENDOR_PREFIXES = 'Win'; + +let cachedRawValue: string | undefined; +let cachedPrefixes: ReadonlySet | undefined; + +/** Return the configured vendor prefixes as a canonical, duplicate-free set. */ +export function springVendorPrefixes(): ReadonlySet { + const rawValue = process.env.GITNEXUS_SPRING_VENDOR_PREFIXES ?? DEFAULT_SPRING_VENDOR_PREFIXES; + if (cachedPrefixes && cachedRawValue === rawValue) return cachedPrefixes; + + cachedRawValue = rawValue; + cachedPrefixes = new Set( + rawValue + .split(',') + .map((prefix) => prefix.trim()) + .filter(Boolean), + ); + return cachedPrefixes; +} + +/** + * Stable metadata value for the route semantics controlled by the prefix list. + * Sorting makes equivalent lists independent of declaration order. + */ +export function springVendorPrefixesKey(): string { + return JSON.stringify([...springVendorPrefixes()].sort()); +} diff --git a/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts b/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts index 1ac21e2f9..c738ce2a4 100644 --- a/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts +++ b/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts @@ -9,6 +9,7 @@ import type Parser from 'tree-sitter'; import type { ExtractedDecoratorRoute } from '../workers/parse-worker.js'; import { intersectSpringHttpMethods, + isClassLevelMappingAnnotation, springAnnotationHttpMethods, unquoteSpringLiteral, } from './spring-shared.js'; @@ -140,19 +141,6 @@ function functionName(node: Parser.SyntaxNode): string | null { return identifier ? unquoteKotlinIdentifier(identifier.text) : null; } -/** - * `springAnnotationHttpMethods` parses Java `{A, B}` collections. - * Translate only Kotlin `method = [A, B]` before delegating. - */ -function kotlinSpringHttpMethods(name: string, annotation: Parser.SyntaxNode): readonly string[] { - if (name !== 'RequestMapping') return springAnnotationHttpMethods(name, annotation.text); - const normalized = annotation.text.replace( - /(\bmethod\s*=\s*)\[([^\]]*)\]/gs, - (_match, assignment: string, values: string) => `${assignment}{${values}}`, - ); - return springAnnotationHttpMethods(name, normalized); -} - function typeName(node: Parser.SyntaxNode): string | null { const identifier = node.children.find((child) => child.type === 'type_identifier'); return identifier ? unquoteKotlinIdentifier(identifier.text) : null; @@ -217,8 +205,8 @@ interface ClassMapping { * mappings, and dynamic expressions fail closed for the whole class. */ function classMapping(annotations: readonly Parser.SyntaxNode[]): ClassMapping | null { - const mappings = annotations.filter( - (annotation) => annotationName(annotation) === 'RequestMapping', + const mappings = annotations.filter((annotation) => + isClassLevelMappingAnnotation(annotationName(annotation) ?? ''), ); if (mappings.length === 0) return { prefix: '', methods: ['*'] }; if (mappings.length !== 1) return null; @@ -241,7 +229,9 @@ function classMapping(annotations: readonly Parser.SyntaxNode[]): ClassMapping | } } - const methods = kotlinSpringHttpMethods('RequestMapping', mapping); + const mappingName = annotationName(mapping); + if (!mappingName) return null; + const methods = springAnnotationHttpMethods(mappingName, mapping.text); return methods.length === 0 ? null : { prefix, methods }; } @@ -274,7 +264,7 @@ export function extractKotlinSpringRoutes( const decoratorName = annotationName(annotation); if (!decoratorName) continue; - const methodMethods = kotlinSpringHttpMethods(decoratorName, annotation); + const methodMethods = springAnnotationHttpMethods(decoratorName, annotation.text); const methods = intersectSpringHttpMethods(ownerMapping.methods, methodMethods); if (methods.length === 0) continue; diff --git a/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts b/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts index d417d5ff4..6224a16c9 100644 --- a/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts +++ b/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts @@ -18,13 +18,14 @@ import type Parser from 'tree-sitter'; import { parseSpringAnnotationArguments } from '../frameworks/spring/annotation-arguments.js'; +import { springVendorPrefixes } from '../frameworks/spring/vendor-prefixes.js'; /** * Spring shortcut method-annotation → HTTP verb. * * `@RequestMapping` is intentionally absent: on a method it carries no implicit * verb (the verb lives in its `method = RequestMethod.X` attribute), and on a - * class it is a URL prefix rather than a route. Callers handle `@RequestMapping` + * class it is a URL prefix rather than a route. Callers handle `RequestMapping` * separately. */ export const METHOD_ANNOTATION_TO_HTTP: Record = { @@ -36,7 +37,46 @@ export const METHOD_ANNOTATION_TO_HTTP: Record = { }; /** - * Parse one `RequestMethod.X` literal or a Java annotation array of literals. + * All recognised Spring mapping-annotation simple names (shortcut + base). + * Sorted longest-first so {@link resolveSpringAnnotationAlias} prefers the most + * specific suffix (e.g. `PostMapping` before any hypothetical shorter overlap). + */ +const SPRING_MAPPING_NAMES: readonly string[] = [ + ...Object.keys(METHOD_ANNOTATION_TO_HTTP), + 'RequestMapping', +].sort((a, b) => b.length - a.length); + +/** + * Resolve a REGISTERED vendor-derived Spring mapping annotation to its base. + * + * Vendor definitions often live in binary dependencies, so their Spring + * meta-annotations cannot be inspected from repository source. Resolution uses + * the conventional `` name instead. + * + * Suffix matching alone accepted unrelated annotations (`@AuditPostMapping` + * produced a phantom route — review P2). Resolution now requires the name to + * be `` with the prefix drawn from a small registry: + * `Win` by default (Winning Health), extendable via + * `GITNEXUS_SPRING_VENDOR_PREFIXES=Win,Acme,Other`. Changing the registry + * invalidates persisted JVM route evidence on the next analysis. Exact-known + * Spring annotation names return `undefined`; callers handle those directly. + */ +export function resolveSpringAnnotationAlias(annotationName: string): string | undefined { + const registeredVendorPrefixes = springVendorPrefixes(); + for (const base of SPRING_MAPPING_NAMES) { + if (annotationName.length > base.length && annotationName.endsWith(base)) { + const prefix = annotationName.slice(0, annotationName.length - base.length); + if (registeredVendorPrefixes.has(prefix)) { + return base; + } + } + } + return undefined; +} + +/** + * Parse one `RequestMethod.X` literal or a Java `{…}` / Kotlin `[…]` array of + * those literals. * An empty array is valid and means Spring's unrestricted/default method set. * Runtime expressions fail closed instead of producing a guessed route. */ @@ -60,16 +100,25 @@ function parseRequestMethodValues(value: string): readonly string[] | null { } trimmed += char; } - const hasOpeningBrace = trimmed.startsWith('{'); - const hasClosingBrace = trimmed.endsWith('}'); - if (hasOpeningBrace !== hasClosingBrace) return null; - const body = hasOpeningBrace ? trimmed.slice(1, -1).trim() : trimmed; + const wrapped = + (trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']')); + if ( + !wrapped && + (trimmed.startsWith('{') || + trimmed.startsWith('[') || + trimmed.endsWith('}') || + trimmed.endsWith(']')) + ) { + return null; + } + const body = wrapped ? trimmed.slice(1, -1).trim() : trimmed; if (body.length === 0) return []; - if (!hasOpeningBrace && body.includes(',')) return null; + if (!wrapped && body.includes(',')) return null; const methods: string[] = []; const parts = body.split(','); - if (hasOpeningBrace && parts[parts.length - 1].trim() === '') parts.pop(); + if (wrapped && parts[parts.length - 1].trim() === '') parts.pop(); for (const rawPart of parts) { const part = rawPart.trim(); const match = @@ -89,14 +138,28 @@ function parseRequestMethodValues(value: string): readonly string[] | null { * one or more static `RequestMethod.X` values; when its `method` member is * absent or an empty array, `'*'` preserves Spring's method-agnostic semantics. * A present but non-static method expression yields no methods (fail closed). + * + * Vendor-derived aliases (e.g. `@WinPostMapping`) are resolved by suffix to + * their base annotation before the above logic applies — see + * {@link resolveSpringAnnotationAlias}. */ export function springAnnotationHttpMethods( annotationName: string, annotationText: string, ): readonly string[] { + // Exact shortcut match (PostMapping → POST, etc.) const shortcut = METHOD_ANNOTATION_TO_HTTP[annotationName]; if (shortcut) return [shortcut]; - if (annotationName !== 'RequestMapping') return []; + + // Resolve vendor alias by suffix (WinPostMapping → PostMapping, etc.) + const base = resolveSpringAnnotationAlias(annotationName) ?? annotationName; + + // Alias of a shortcut annotation + const aliasShortcut = METHOD_ANNOTATION_TO_HTTP[base]; + if (aliasShortcut) return [aliasShortcut]; + + // Direct or aliased @RequestMapping: parse the method= attribute + if (base !== 'RequestMapping') return []; const args = parseSpringAnnotationArguments(annotationText); if (args === null) return []; @@ -109,6 +172,19 @@ export function springAnnotationHttpMethods( return methods.length > 0 ? methods : ['*']; } +/** + * True when an annotation name is a class-level request-mapping annotation — + * either Spring's own `@RequestMapping` or a registered vendor alias that + * resolves to it (`@WinRequestMapping`). Class-level handling in both the + * group extractor and the ingestion route extractor routes through this + * predicate so vendor aliases get the same prefix/constraint semantics as + * the base annotation (review P1). + */ +export function isClassLevelMappingAnnotation(annotationName: string): boolean { + if (annotationName === 'RequestMapping') return true; + return resolveSpringAnnotationAlias(annotationName) === 'RequestMapping'; +} + /** Intersect class- and method-level Spring mapping constraints. */ export function intersectSpringHttpMethods( classMethods: readonly string[], diff --git a/gitnexus/src/core/ingestion/route-extractors/spring.ts b/gitnexus/src/core/ingestion/route-extractors/spring.ts index 36828a1f5..701e7c03f 100644 --- a/gitnexus/src/core/ingestion/route-extractors/spring.ts +++ b/gitnexus/src/core/ingestion/route-extractors/spring.ts @@ -27,6 +27,7 @@ import { springAnnotationHttpMethods, isRouteMemberKey, findEnclosingType, + isClassLevelMappingAnnotation, unquoteSpringLiteral, type SharedSpringType, } from './spring-shared.js'; @@ -192,7 +193,10 @@ export function extractSpringRoutes( if (!annNode || !node || (!valueNode && !valueExprNode)) continue; const capturedAnnotationName = annNode.text.split('.').pop() ?? annNode.text; - if (node.type === 'class_declaration' && capturedAnnotationName === 'RequestMapping') { + if ( + node.type === 'class_declaration' && + isClassLevelMappingAnnotation(capturedAnnotationName) + ) { if (!isRouteMemberKey(keyNode)) continue; if (!valueNode) { classesWithUnfoldablePrefix.add(node.id); @@ -437,12 +441,14 @@ function annotationHasRouteMember(ann: Parser.SyntaxNode): boolean { /** Static class/interface-level RequestMapping method constraint, or wildcard by default. */ function typeRequestMethods(typeNode: Parser.SyntaxNode): readonly string[] { - const mappings = declarationAnnotations(typeNode).filter( - (ann) => annotationName(ann) === 'RequestMapping', + const mappings = declarationAnnotations(typeNode).filter((ann) => + isClassLevelMappingAnnotation(annotationName(ann) ?? ''), ); if (mappings.length === 0) return ['*']; if (mappings.length !== 1) return []; - return springAnnotationHttpMethods('RequestMapping', mappings[0].text); + const mappingName = annotationName(mappings[0]); + if (!mappingName) return []; + return springAnnotationHttpMethods(mappingName, mappings[0].text); } function annotationRoutePathsOrDefault(ann: Parser.SyntaxNode): string[] { @@ -455,7 +461,8 @@ function annotationRoutePathsOrDefault(ann: Parser.SyntaxNode): string[] { function typeClassPrefixes(typeNode: Parser.SyntaxNode): string[] { const prefixes: string[] = []; for (const ann of declarationAnnotations(typeNode)) { - if (annotationName(ann) === 'RequestMapping') prefixes.push(...annotationRoutePaths(ann)); + if (isClassLevelMappingAnnotation(annotationName(ann) ?? '')) + prefixes.push(...annotationRoutePaths(ann)); } return prefixes; } diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index bcadab3ec..be591792e 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -190,22 +190,13 @@ import { isSpringBeanCandidateSourceFile } from './ingestion/frameworks/spring/b import { isSpringBeanFactoryDeclaration } from './ingestion/frameworks/spring/bean-factories.js'; import { SPRING_CONFIG_UNRESOLVED_PREFIX } from './ingestion/frameworks/spring/config-bindings.js'; import { classifySpringConfigFile } from './ingestion/pipeline-phases/spring-config.js'; +import { SPRING_ROUTE_BINDINGS_FEATURE } from './ingestion/frameworks/spring/analysis-features.js'; +import { springVendorPrefixesKey } from './ingestion/frameworks/spring/vendor-prefixes.js'; import { - SPRING_AOP_FEATURE, - SPRING_BEAN_INVENTORY_FEATURE, - SPRING_CONDITIONALS_FEATURE, - SPRING_NON_HTTP_HANDLERS_FEATURE, -} from './ingestion/frameworks/spring/analysis-features.js'; -import { - JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, - JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, - SPRING_CONFIG_BINDINGS_FEATURE, -} from './ingestion/languages/java/analysis-features.js'; -import { - CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, findAnalysisFeatureMismatches, resolveAnalysisFeatureVersions, } from './analysis-features.js'; +import { ANALYSIS_FEATURES } from './analysis-feature-registry.js'; import { analyzerRunnerIdentitiesEqual, finalizeAnalyzerRunnerIdentity, @@ -247,17 +238,6 @@ import type { EmbeddingCheckpoint } from './embedding-checkpoint.js'; const stripControlCharacters = (msg: string): string => msg.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, ''); -const ANALYSIS_FEATURES = [ - CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, - SPRING_AOP_FEATURE, - SPRING_BEAN_INVENTORY_FEATURE, - SPRING_CONDITIONALS_FEATURE, - SPRING_NON_HTTP_HANDLERS_FEATURE, - SPRING_CONFIG_BINDINGS_FEATURE, - JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, - JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, -] as const; - interface PersistedFrameworkAnnotationRow { readonly id?: unknown; readonly frameworkAnnotations?: unknown; @@ -1597,6 +1577,20 @@ async function runFullAnalysisInner( analysisFeatureMismatchLogged = true; } + const currentSpringVendorPrefixes = springVendorPrefixesKey(); + const persistedRouteBindings = existingMeta?.analysisFeatures?.[SPRING_ROUTE_BINDINGS_FEATURE.id]; + if ( + existingMeta && + persistedRouteBindings === SPRING_ROUTE_BINDINGS_FEATURE.version && + existingMeta.springVendorPrefixes !== currentSpringVendorPrefixes + ) { + log( + 'Spring vendor mapping prefixes changed; forcing a full rebuild so persisted Route ' + + 'evidence matches the configured aliases.', + ); + options = { ...options, force: true }; + } + // Analyzer provenance is part of freshness, not merely diagnostics. A // same-commit fast path must not preserve metadata produced by an older, // malformed, or dependency/native-different runner. Force a real rebuild so @@ -3911,6 +3905,7 @@ async function runFullAnalysisInner( ? existingMeta?.undecidedInterfaceSatisfaction : summarizeUndecidedSatisfaction(pipelineResult.undecidedSatisfaction), analysisFeatures: currentAnalysisFeatures, + springVendorPrefixes: currentSpringVendorPrefixes, // Always stamped with the live resolved mode (#2331/#2339) — unlike // `pdg` below, 'none' is a meaningful value to compare, not an // absence, so this is never conditionally omitted. diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 3d6c7d376..2b8d453e6 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -202,6 +202,11 @@ export interface RepoMeta { * containing relevant source files. */ analysisFeatures?: Record; + /** + * Canonical registered-prefix list used to resolve vendor Spring mapping + * annotations. A changed value invalidates persisted JVM Route evidence. + */ + springVendorPrefixes?: string; /** * The resolved GITNEXUS_FTS_CJK_SEGMENTATION mode ('none' | 'bigram') the * existing index's content/description columns were last written under diff --git a/gitnexus/test/unit/analysis-features.test.ts b/gitnexus/test/unit/analysis-features.test.ts index 415026be1..08d4fd24c 100644 --- a/gitnexus/test/unit/analysis-features.test.ts +++ b/gitnexus/test/unit/analysis-features.test.ts @@ -5,35 +5,14 @@ import { resolveAnalysisFeatureVersions, type AnalysisFeatureDescriptor, } from '../../src/core/analysis-features.js'; -import { - SPRING_AOP_FEATURE, - SPRING_BEAN_INVENTORY_FEATURE, - SPRING_CONDITIONALS_FEATURE, - SPRING_NON_HTTP_HANDLERS_FEATURE, -} from '../../src/core/ingestion/frameworks/spring/analysis-features.js'; -import { - JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, - JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, - SPRING_CONFIG_BINDINGS_FEATURE, -} from '../../src/core/ingestion/languages/java/analysis-features.js'; - -const FEATURES = [ - CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, - SPRING_AOP_FEATURE, - SPRING_BEAN_INVENTORY_FEATURE, - SPRING_CONDITIONALS_FEATURE, - SPRING_NON_HTTP_HANDLERS_FEATURE, - SPRING_CONFIG_BINDINGS_FEATURE, - JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, - JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, -] as const; +import { ANALYSIS_FEATURES } from '../../src/core/analysis-feature-registry.js'; describe('analysis feature versions', () => { it('separates the global Class schema capability from JVM-only Bean evidence', () => { - expect(resolveAnalysisFeatureVersions(FEATURES, ['src/app.ts'])).toEqual({ + expect(resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, ['src/app.ts'])).toEqual({ 'graph.class-framework-annotations': 1, }); - expect(resolveAnalysisFeatureVersions(FEATURES, ['src/App.java'])).toEqual({ + expect(resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, ['src/App.java'])).toEqual({ 'graph.class-framework-annotations': 1, 'java.heritage-captures': 1, 'java.record-component-accessors': 1, @@ -42,24 +21,27 @@ describe('analysis feature versions', () => { 'spring.conditionals-auto-configuration': 1, 'spring.config-bindings': 2, 'spring.non-http-handlers': 1, + 'spring.route-bindings': 2, }); - expect(resolveAnalysisFeatureVersions(FEATURES, ['src/App.kt'])).toEqual({ + expect(resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, ['src/App.kt'])).toEqual({ 'graph.class-framework-annotations': 1, 'spring.aop-advice': 1, 'spring.bean-inventory': 2, 'spring.conditionals-auto-configuration': 1, 'spring.config-bindings': 2, 'spring.non-http-handlers': 1, + 'spring.route-bindings': 2, }); - expect(resolveAnalysisFeatureVersions(FEATURES, ['BUILD.GRADLE.KTS'])).toEqual({ + expect(resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, ['BUILD.GRADLE.KTS'])).toEqual({ 'graph.class-framework-annotations': 1, 'spring.aop-advice': 1, 'spring.bean-inventory': 2, 'spring.conditionals-auto-configuration': 1, 'spring.non-http-handlers': 1, + 'spring.route-bindings': 2, }); expect( - resolveAnalysisFeatureVersions(FEATURES, [ + resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, [ 'src/main/resources/application-local.yml', 'README.md', ]), @@ -68,7 +50,7 @@ describe('analysis feature versions', () => { 'spring.config-bindings': 2, }); expect( - resolveAnalysisFeatureVersions(FEATURES, [ + resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, [ 'src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports', ]), ).toEqual({ diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index 3e7b89c07..f28992310 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -48,7 +48,9 @@ import { SPRING_BEAN_INVENTORY_FEATURE, SPRING_CONDITIONALS_FEATURE, SPRING_NON_HTTP_HANDLERS_FEATURE, + SPRING_ROUTE_BINDINGS_FEATURE, } from '../../src/core/ingestion/frameworks/spring/analysis-features.js'; +import { springVendorPrefixesKey } from '../../src/core/ingestion/frameworks/spring/vendor-prefixes.js'; import { decodeSpringAopReason, SPRING_AOP_EVIDENCE_ID_PREFIX, @@ -799,6 +801,7 @@ describe('runFullAnalysis — incremental orchestration', () => { [SPRING_CONDITIONALS_FEATURE.id]: SPRING_CONDITIONALS_FEATURE.version, [SPRING_CONFIG_BINDINGS_FEATURE.id]: SPRING_CONFIG_BINDINGS_FEATURE.version, [SPRING_NON_HTTP_HANDLERS_FEATURE.id]: SPRING_NON_HTTP_HANDLERS_FEATURE.version, + [SPRING_ROUTE_BINDINGS_FEATURE.id]: SPRING_ROUTE_BINDINGS_FEATURE.version, }); await saveMeta(storagePath, withoutAnalysisFeature(meta!, SPRING_BEAN_INVENTORY_FEATURE.id)); @@ -819,12 +822,45 @@ describe('runFullAnalysis — incremental orchestration', () => { [SPRING_CONDITIONALS_FEATURE.id]: SPRING_CONDITIONALS_FEATURE.version, [SPRING_CONFIG_BINDINGS_FEATURE.id]: SPRING_CONFIG_BINDINGS_FEATURE.version, [SPRING_NON_HTTP_HANDLERS_FEATURE.id]: SPRING_NON_HTTP_HANDLERS_FEATURE.version, + [SPRING_ROUTE_BINDINGS_FEATURE.id]: SPRING_ROUTE_BINDINGS_FEATURE.version, }); } finally { await repo.cleanup(); } }, 300_000); + it('rebuilds a JVM index when the registered Spring vendor prefixes change', async () => { + const repo = await setupSpringBeanIncrementalRepo(); + try { + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Win'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }); + const { storagePath } = getStoragePaths(repo.dbPath); + expect((await loadMeta(storagePath))?.springVendorPrefixes).toBe(springVendorPrefixesKey()); + + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Acme,Win'); + const logs: string[] = []; + const rebuilt = await runFullAnalysis( + repo.dbPath, + { skipAgentsMd: true }, + { onProgress: () => {}, onLog: (message) => logs.push(message) }, + ); + + expect(rebuilt.alreadyUpToDate).toBeUndefined(); + expect(logs.join('\n')).toContain('Spring vendor mapping prefixes changed'); + expect((await loadMeta(storagePath))?.springVendorPrefixes).toBe(springVendorPrefixesKey()); + + const steady = await runFullAnalysis( + repo.dbPath, + { skipAgentsMd: true }, + { onProgress: () => {} }, + ); + expect(steady.alreadyUpToDate).toBe(true); + } finally { + await repo.cleanup(); + } + }, 300_000); + it('a config-only index missing Spring config evidence rebuilds and restores the scoped stamp', async () => { const repo = await setupSpringConfigIncrementalRepo(); try { @@ -927,6 +963,7 @@ describe('runFullAnalysis — incremental orchestration', () => { [SPRING_CONDITIONALS_FEATURE.id]: SPRING_CONDITIONALS_FEATURE.version, [SPRING_CONFIG_BINDINGS_FEATURE.id]: SPRING_CONFIG_BINDINGS_FEATURE.version, [SPRING_NON_HTTP_HANDLERS_FEATURE.id]: SPRING_NON_HTTP_HANDLERS_FEATURE.version, + [SPRING_ROUTE_BINDINGS_FEATURE.id]: SPRING_ROUTE_BINDINGS_FEATURE.version, }); } finally { await repo.cleanup(); diff --git a/gitnexus/test/unit/kotlin-spring-route-ingestion.test.ts b/gitnexus/test/unit/kotlin-spring-route-ingestion.test.ts index 7ec94df5a..51c2225dd 100644 --- a/gitnexus/test/unit/kotlin-spring-route-ingestion.test.ts +++ b/gitnexus/test/unit/kotlin-spring-route-ingestion.test.ts @@ -4,7 +4,7 @@ * The Kotlin grammar is optional. Importing the extractor itself must not load * that grammar; only this guarded test setup does. */ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import Parser from 'tree-sitter'; import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js'; import { extractKotlinSpringRoutes } from '../../src/core/ingestion/route-extractors/kotlin-spring.js'; @@ -29,16 +29,23 @@ if (Kotlin) parser.setLanguage(Kotlin as Parser.Language); const parse = (source: string): Parser.Tree => parser.parse(source); const describeKotlin = Kotlin ? describe : describe.skip; -function constantsOf(files: Record): RepoConstants { - return new Map( - Object.entries(files).map(([filePath, source]) => [ - filePath, - extractKotlinModuleConstants(parse(source)), - ]), - ); -} - describeKotlin('extractKotlinSpringRoutes', () => { + beforeEach(() => { + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Win'); + }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + function constantsOf(files: Record): RepoConstants { + return new Map( + Object.entries(files).map(([filePath, source]) => [ + filePath, + extractKotlinModuleConstants(parse(source)), + ]), + ); + } + it('extracts direct RestController functions with independent class prefixes and handlers', () => { const routes = extractKotlinSpringRoutes( parse(` @@ -627,4 +634,134 @@ class PetsController { expect(ingestion).toEqual(group); }); + + it('resolves vendor-derived mapping aliases like Java (WinGetMapping / WinRequestMapping)', () => { + expect(KOTLIN_HTTP_PLUGIN).not.toBeNull(); + if (!KOTLIN_HTTP_PLUGIN) throw new Error('expected Kotlin HTTP plugin'); + const source = ` +@RestController +@WinRequestMapping("/vendor") +class VendorController { + @WinGetMapping("/users") + fun users(): String = "ok" +} +`; + const tree = parse(source); + const ingestion = extractKotlinSpringRoutes(tree, 'VendorController.kt'); + expect(ingestion).toHaveLength(1); + expect(ingestion[0]?.httpMethod).toBe('GET'); + expect(ingestion[0]?.prefix).toBe('/vendor'); + expect(ingestion[0]?.routePath).toBe('/users'); + + const group = KOTLIN_HTTP_PLUGIN.scan(tree, undefined, 'VendorController.kt').filter( + (detection) => detection.role === 'provider', + ); + expect(group).toEqual( + expect.arrayContaining([expect.objectContaining({ method: 'GET', path: '/vendor/users' })]), + ); + }); + + it('normalizes Kotlin method arrays for aliased RequestMapping annotations', () => { + expect(KOTLIN_HTTP_PLUGIN).not.toBeNull(); + if (!KOTLIN_HTTP_PLUGIN) throw new Error('expected Kotlin HTTP plugin'); + const source = ` +@RestController +@WinRequestMapping("/vendor") +class VendorController { + @WinRequestMapping(path = "/inspect", method = [RequestMethod.GET, RequestMethod.HEAD]) + fun inspect(): String = "ok" +} +`; + const tree = parse(source); + const ingestion = new Set( + extractKotlinSpringRoutes(tree, 'VendorController.kt').map( + (route) => `${route.httpMethod} ${joinPath(route.prefix ?? '', route.routePath)}`, + ), + ); + const group = new Set( + KOTLIN_HTTP_PLUGIN.scan(tree, undefined, 'VendorController.kt') + .filter((detection) => detection.role === 'provider') + .map((detection) => `${detection.method} ${detection.path}`), + ); + + expect(ingestion).toEqual(new Set(['GET /vendor/inspect', 'HEAD /vendor/inspect'])); + expect(group).toEqual(ingestion); + }); + + it('applies aliased class-level Kotlin method arrays to handler routes', () => { + expect(KOTLIN_HTTP_PLUGIN).not.toBeNull(); + if (!KOTLIN_HTTP_PLUGIN) throw new Error('expected Kotlin HTTP plugin'); + const tree = parse(` +@RestController +@WinRequestMapping(path = "/vendor", method = [RequestMethod.GET, RequestMethod.HEAD]) +class VendorController { + @WinRequestMapping("/inspect") + fun inspect(): String = "ok" +} +`); + const ingestion = new Set( + extractKotlinSpringRoutes(tree, 'VendorController.kt').map( + (route) => `${route.httpMethod} ${joinPath(route.prefix ?? '', route.routePath)}`, + ), + ); + const group = new Set( + KOTLIN_HTTP_PLUGIN.scan(tree, undefined, 'VendorController.kt') + .filter((detection) => detection.role === 'provider') + .map((detection) => `${detection.method} ${detection.path}`), + ); + + expect(ingestion).toEqual(new Set(['GET /vendor/inspect', 'HEAD /vendor/inspect'])); + expect(group).toEqual(ingestion); + }); + + it('keeps aliased class method constraints in inherited group contracts', () => { + expect(KOTLIN_HTTP_PLUGIN?.scanProject).toBeDefined(); + if (!KOTLIN_HTTP_PLUGIN?.scanProject) throw new Error('expected Kotlin project scanner'); + const tree = parse(` +@WinRequestMapping(path = "/contract", method = [RequestMethod.GET]) +interface Contract { + @WinRequestMapping(path = "/items", method = [RequestMethod.GET, RequestMethod.POST]) + fun inspect(): String +} + +@RestController +@WinRequestMapping("/impl") +class VendorController : Contract { + override fun inspect(): String = "ok" +} +`); + + const detections = KOTLIN_HTTP_PLUGIN.scanProject([ + { filePath: 'VendorController.kt', tree }, + ]).flatMap((file) => file.detections); + + expect(detections).toEqual([ + expect.objectContaining({ + role: 'provider', + method: 'GET', + path: '/impl/contract/items', + }), + ]); + }); + + it('does not treat unregistered suffix annotations as Kotlin routes', () => { + const source = ` +@RestController +class AuditController { + @AuditPostMapping("/audit") + fun audit(): String = "x" + + @AuditRequestMapping(path = "/request", method = [RequestMethod.POST]) + fun request(): String = "x" +} +`; + const tree = parse(source); + expect(extractKotlinSpringRoutes(tree, 'AuditController.kt')).toHaveLength(0); + expect(KOTLIN_HTTP_PLUGIN).not.toBeNull(); + expect( + KOTLIN_HTTP_PLUGIN?.scan(tree, undefined, 'AuditController.kt').filter( + (detection) => detection.role === 'provider', + ), + ).toEqual([]); + }); }); diff --git a/gitnexus/test/unit/spring-vendor-annotation-alias.test.ts b/gitnexus/test/unit/spring-vendor-annotation-alias.test.ts new file mode 100644 index 000000000..317b0beb6 --- /dev/null +++ b/gitnexus/test/unit/spring-vendor-annotation-alias.test.ts @@ -0,0 +1,297 @@ +/** + * Unit test: vendor-derived Spring mapping annotation alias resolution. + * + * Frameworks wrap Spring's built-in annotations with company-specific variants + * (e.g. Winning Health's `@WinPostMapping`). The tree-sitter query captures + * these annotations like any other, but `springAnnotationHttpMethods` must + * resolve them to the correct HTTP verb via suffix matching. + * + * These tests cover: + * 1. `resolveSpringAnnotationAlias` directly (unit) + * 2. `springAnnotationHttpMethods` with aliased annotations (unit) + * 3. End-to-end `extractSpringRoutes` with a fixture using vendor annotations + * 4. Parity: both ingestion and group extractors surface the same routes + */ +import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { + resolveSpringAnnotationAlias, + springAnnotationHttpMethods, +} from '../../src/core/ingestion/route-extractors/spring-shared.js'; +import { extractSpringRoutes } from '../../src/core/ingestion/route-extractors/spring.js'; +import { JAVA_HTTP_PLUGIN } from '../../src/core/group/extractors/http-patterns/java.js'; +import { normalizeExtractedRoutePath } from '../../src/core/ingestion/route-extractors/route-path.js'; +import { springVendorPrefixesKey } from '../../src/core/ingestion/frameworks/spring/vendor-prefixes.js'; + +function parse(code: string): Parser.Tree { + const parser = new Parser(); + parser.setLanguage(Java); + return parser.parse(code); +} + +beforeEach(() => { + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Win'); +}); +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('resolveSpringAnnotationAlias', () => { + it('returns undefined for exact built-in shortcut annotations', () => { + expect(resolveSpringAnnotationAlias('PostMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('GetMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('PutMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('DeleteMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('PatchMapping')).toBeUndefined(); + }); + + it('returns undefined for exact RequestMapping', () => { + expect(resolveSpringAnnotationAlias('RequestMapping')).toBeUndefined(); + }); + + it('returns undefined for unrelated annotations', () => { + expect(resolveSpringAnnotationAlias('Override')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('Autowired')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('Component')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('Data')).toBeUndefined(); + }); + + it('resolves vendor shortcut annotations by suffix', () => { + expect(resolveSpringAnnotationAlias('WinPostMapping')).toBe('PostMapping'); + expect(resolveSpringAnnotationAlias('WinGetMapping')).toBe('GetMapping'); + expect(resolveSpringAnnotationAlias('WinPutMapping')).toBe('PutMapping'); + expect(resolveSpringAnnotationAlias('WinDeleteMapping')).toBe('DeleteMapping'); + expect(resolveSpringAnnotationAlias('WinPatchMapping')).toBe('PatchMapping'); + }); + + it('resolves vendor RequestMapping variants', () => { + expect(resolveSpringAnnotationAlias('WinRequestMapping')).toBe('RequestMapping'); + }); + + it('ignores unregistered vendor prefixes (review: suffix-only accepted @AuditPostMapping)', () => { + // Suffix matching alone produced phantom routes from unrelated + // annotations like @AuditPostMapping — resolution now requires a + // registered prefix (Win by default). + expect(resolveSpringAnnotationAlias('AuditPostMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('CompanyPostMapping')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('XyzGetMapping')).toBeUndefined(); + }); + + it('does not match annotations that merely contain a mapping name', () => { + expect(resolveSpringAnnotationAlias('PostMappingHelper')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('GetMappingInfo')).toBeUndefined(); + expect(resolveSpringAnnotationAlias('PreMapping')).toBeUndefined(); + }); +}); + +describe('Spring vendor prefix freshness', () => { + it('canonicalizes equivalent lists regardless of order and duplicates', () => { + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', ' Win,Acme,Win '); + const first = springVendorPrefixesKey(); + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Acme,Win'); + const second = springVendorPrefixesKey(); + + expect(first).toBe('["Acme","Win"]'); + expect(second).toBe(first); + expect(first).not.toBe('["Win"]'); + }); +}); + +describe('springAnnotationHttpMethods with vendor aliases', () => { + it('resolves WinPostMapping to POST', () => { + expect(springAnnotationHttpMethods('WinPostMapping', '@WinPostMapping("/api")')).toEqual([ + 'POST', + ]); + }); + + it('resolves WinGetMapping to GET', () => { + expect(springAnnotationHttpMethods('WinGetMapping', '@WinGetMapping("/api")')).toEqual(['GET']); + }); + + it('resolves WinDeleteMapping to DELETE', () => { + expect(springAnnotationHttpMethods('WinDeleteMapping', '@WinDeleteMapping("/api")')).toEqual([ + 'DELETE', + ]); + }); + + it('resolves WinRequestMapping without method attribute to wildcard', () => { + expect(springAnnotationHttpMethods('WinRequestMapping', '@WinRequestMapping("/api")')).toEqual([ + '*', + ]); + }); + + it('resolves WinRequestMapping with method attribute', () => { + const text = '@WinRequestMapping(value = "/api", method = RequestMethod.POST)'; + expect(springAnnotationHttpMethods('WinRequestMapping', text)).toEqual(['POST']); + }); + + it('accepts Kotlin collection syntax for RequestMapping method arrays', () => { + const text = + '@WinRequestMapping(value = "/api", method = [RequestMethod.GET, RequestMethod.HEAD])'; + expect(springAnnotationHttpMethods('WinRequestMapping', text)).toEqual(['GET', 'HEAD']); + }); + + it('fail-closes mismatched RequestMapping method collection delimiters', () => { + const text = '@WinRequestMapping(method = {RequestMethod.GET])'; + expect(springAnnotationHttpMethods('WinRequestMapping', text)).toEqual([]); + }); + + it('returns empty for unrelated annotations', () => { + expect(springAnnotationHttpMethods('Component', '@Component')).toEqual([]); + expect(springAnnotationHttpMethods('Override', '@Override')).toEqual([]); + }); +}); + +describe('extractSpringRoutes with vendor annotations', () => { + it('extracts routes from a controller using @Win annotations', () => { + const tree = parse(` +package com.winning.opt.controller; + +@RestController +@RequestMapping("/api/opt") +public class OrderController { + @WinPostMapping("/create") + public String create() { return "{}"; } + + @WinGetMapping("/query") + public String query() { return "[]"; } + + @WinPostMapping(value = "/update") + public String update() { return "{}"; } +} +`); + + const routes = extractSpringRoutes(tree, 'OrderController.java'); + expect(routes).toHaveLength(3); + + const postRoutes = routes.filter((r) => r.httpMethod === 'POST'); + expect(postRoutes).toHaveLength(2); + const postPaths = postRoutes.map((r) => r.routePath).sort(); + expect(postPaths).toEqual(['/create', '/update']); + for (const r of postRoutes) { + expect(r.prefix).toBe('/api/opt'); + } + + const getRoute = routes.find((r) => r.httpMethod === 'GET')!; + expect(getRoute.routePath).toBe('/query'); + expect(getRoute.prefix).toBe('/api/opt'); + }); + + it('extracts routes when vendor and standard annotations are mixed', () => { + const tree = parse(` +@RestController +@RequestMapping("/api/mix") +public class MixedController { + @WinPostMapping("/win-create") + public String winCreate() { return "{}"; } + + @PostMapping("/std-create") + public String stdCreate() { return "{}"; } + + @GetMapping("/std-get") + public String stdGet() { return "[]"; } +} +`); + + const routes = extractSpringRoutes(tree, 'MixedController.java'); + expect(routes).toHaveLength(3); + + const paths = routes.map((r) => r.routePath).sort(); + expect(paths).toEqual(['/std-create', '/std-get', '/win-create']); + }); + + it('ingestion and group extractors agree on vendor annotation routes', () => { + const tree = parse(` +@RestController +@RequestMapping("/api/parity") +public class ParityController { + @WinPostMapping("/create") + public String create() { return "{}"; } + + @WinGetMapping("/query") + public String query() { return "[]"; } +} +`); + + const ingestionRoutes = new Set( + extractSpringRoutes(tree, 'ParityController.java').map( + (r) => `${r.httpMethod} ${normalizeExtractedRoutePath(r.routePath, r.prefix ?? null)}`, + ), + ); + + const groupRoutes = new Set( + JAVA_HTTP_PLUGIN.scan(tree) + .filter((d) => d.role === 'provider') + .map((d) => `${d.method} ${normalizeExtractedRoutePath(d.path, null)}`), + ); + + expect([...ingestionRoutes].sort()).toEqual([...groupRoutes].sort()); + expect([...ingestionRoutes].sort()).toEqual([ + 'GET /api/parity/query', + 'POST /api/parity/create', + ]); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// Review regressions (magyargergo, 2026-08-29) +// ═══════════════════════════════════════════════════════════════════════════ + +describe('review regressions: class-level aliases', () => { + it('P1: isClassLevelMappingAnnotation accepts @WinRequestMapping like @RequestMapping', async () => { + const { isClassLevelMappingAnnotation } = + await import('../../src/core/ingestion/route-extractors/spring-shared.js'); + expect(isClassLevelMappingAnnotation('RequestMapping')).toBe(true); + expect(isClassLevelMappingAnnotation('WinRequestMapping')).toBe(true); + expect(isClassLevelMappingAnnotation('WinPostMapping')).toBe(false); + expect(isClassLevelMappingAnnotation('AuditRequestMapping')).toBe(false); + expect(isClassLevelMappingAnnotation('GetMapping')).toBe(false); + }); + + it('P1: vendor class prefix flows into route paths (@WinRequestMapping + @WinGetMapping)', () => { + const tree = parse(` +@WinRequestMapping("/vendor") +public class VendorController { + @WinGetMapping("/users") + public String list() { return "ok"; } +} +`); + const routes = extractSpringRoutes(tree, 'VendorController.java'); + expect(routes).toHaveLength(1); + // Class prefix /vendor comes from the aliased @WinRequestMapping — the + // exact path the old exact-match-only class handling missed (review P1). + expect(routes[0].prefix).toBe('/vendor'); + expect(routes[0].routePath).toBe('/users'); + expect(routes[0].httpMethod).toBe('GET'); + expect( + JAVA_HTTP_PLUGIN.scan(tree) + .filter((detection) => detection.role === 'provider') + .map((detection) => `${detection.method} ${detection.path}`), + ).toEqual(['GET /vendor/users']); + }); + + it('P2: unregistered suffix no longer emits a phantom route (end-to-end)', () => { + const tree = parse(` +public class AuditController { + @AuditPostMapping("/audit") + public String audit() { return "x"; } +} +`); + const routes = extractSpringRoutes(tree, 'AuditController.java'); + expect(routes).toHaveLength(0); + expect( + JAVA_HTTP_PLUGIN.scan(tree).filter((detection) => detection.role === 'provider'), + ).toEqual([]); + }); + + it('P2: extra vendor prefixes can be registered via env', () => { + vi.stubEnv('GITNEXUS_SPRING_VENDOR_PREFIXES', 'Win,Acme'); + try { + expect(resolveSpringAnnotationAlias('AcmePostMapping')).toBe('PostMapping'); + expect(resolveSpringAnnotationAlias('OtherPostMapping')).toBeUndefined(); + } finally { + vi.unstubAllEnvs(); + } + }); +}); From b15ff2d888949e16834cb341e81f9345931dfc96 Mon Sep 17 00:00:00 2001 From: glier Date: Thu, 3 Sep 2026 01:01:15 +0300 Subject: [PATCH 05/21] feat(ingestion): mint Destination nodes from AsyncAPI 3.x documents (#3140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ingestion): read AsyncAPI 3.x documents into broker addresses Adds a format-driven reader that turns the `operations[]` entries of an AsyncAPI 3.x document into (broker, address, direction) triples, plus the protocol-to-broker map behind it. Nothing consumes it yet. The reader lives outside `frameworks/spring/` on purpose, like `destination-key.ts` and for the same reason: an AsyncAPI document is a published artifact emitted by generators across several language toolchains and written by hand as often as generated. The entry criterion is therefore the document format -- a root `asyncapi` key -- and never the generator. AsyncAPI 2.x is refused under its own countable reason rather than mapped. Its `publish`/`subscribe` are inverted relative to 3.x `send`/`receive`, so a naive mapping reverses every direction in the async graph while leaving it connected: nothing fails, the arrows simply point the wrong way. A silent skip would be indistinguishable from "this service publishes no document", which is the one thing the refusal count has to be able to tell us. The broker is read twice over -- from the operation's bindings and from its channel's server protocol -- and the two readings must agree. A destination keyed on the wrong broker joins a stranger, and with the document contradicting itself there is no way to tell which reading is right, so the operation is refused rather than decided by a coin flip. An unmapped protocol passes through as its own literal instead of being dropped, because `destinationNodeKey` takes a plain string precisely so a non-Spring caller can attest to a broker Spring has no member for. Co-Authored-By: Claude Opus 5 (1M context) * feat(cli): add --asyncapi-spec, an explicit path to AsyncAPI documents Threads an `asyncApiSpecPath` option from the CLI, the server analyze endpoint, and the programmatic entry through to `PipelineOptions`. Nothing reads it yet; the reader added in the previous commit is still unwired. Shaped deliberately after `springActuatorPath`, the existing option for an out-of-band artifact: an explicit local path, accepting a directory or a single file, resolved against the repository root so a committed `docs/asyncapi` and an absolute cache populated by something else are both natural, and `undefined` keeping the feature entirely off. Mirroring that option rather than inventing a mechanism is what lets a downstream consumer point the reader at documents fetched out of band without patching a file here. `analyze --watch` REJECTS the flag, exactly as it rejects --spring-actuator. The watcher reacts to source changes and nothing watches a document directory, so honouring it there would read the documents once and then serve a stale answer for the rest of the session -- worse than refusing, because it looks like it worked. Additive only: 49 inserted lines, no deletions and no modified lines. Every new interface member is optional and every forward is an object-literal spread of an undefined value, so with the option unset the analyzer takes byte-identical paths. Co-Authored-By: Claude Opus 5 (1M context) * feat(ingestion): mint Destination nodes from AsyncAPI documents Wires the reader into the destinations phase. With `--asyncapi-spec` set, every `send` operation emits PUBLISHES_TO and every `receive` emits CONSUMES_FROM against the ordinary resolved `Destination` node -- same key, same `address` property -- so a document and a source site that name one address on one broker land on ONE node and the two halves of a conversation meet. Verified end to end: an address named only in a document is minted with the right broker and direction, and an address a source site already resolved stays a single node with its own `literal` provenance while the same address on a different broker stays separate. This claims only what a document states -- that the service talks to that address, on that broker, in that direction -- and never which method does it. The addresses in one document partition by (broker, action) into buckets that usually hold more than one operation, so any assignment past a bucket of size one is a heuristic, and a wrong one attaches a real address to the wrong handler: a false connection wearing the clothes of a resolved one. The edge therefore starts at the document, not at a callable. That is weaker than a source-derived edge and worth having anyway, because it is available where the source supplies nothing at all -- a programmatically registered listener, a broker with no patterns here, a language whose messaging idiom nobody has taught this codebase yet. Documents are read even when the source pass found no messaging, which is why the early return had to move: a repository whose brokers are invisible to the patterns is precisely the case a published document covers, and an early return keyed on source sites skipped the documents exactly there. Their counters are kept in their own block rather than folded into the existing ones. `refusalsByReason` is the denominator of the SOURCE unresolved fraction, and a mistyped specification directory must not be able to make the source look worse than it is. The block is absent -- not zeroed -- when no path was configured, so "not asked for" stays distinguishable from "asked for and found nothing"; those need different answers from an operator and one zero cannot say which happened. The direction assertion is the one that matters and it is pinned by type, not by existence: inverting the mapping in the source tree fails exactly one test, because every other assertion passes identically under both readings. Co-Authored-By: Claude Opus 5 (1M context) * docs: document --asyncapi-spec and why step 4 of the cascade stays empty Adds the flag to both READMEs and to the three byte-identical copies of the CLI skill, which a sync test pins together. Also rewrites the note on the `specification` seam in the address cascade. It said "nothing supplies it today", which was true and is now misleading: a reader exists, and the hook is still unsupplied because of a decision rather than for want of one. A document names addresses; it does not name the method that uses one. To hand an address to a particular candidate something must choose which of the document's operations belongs to it, and the only division both sides agree on -- (broker, action) -- leaves buckets that usually hold more than one operation. On a real generated document exactly one bucket of four was unambiguous. Every assignment past a bucket of size one is a heuristic, and a wrong one puts a REAL address on a joining node under the wrong site: a false connection wearing the clothes of a resolved one, which is the outcome the keying rule exists to prevent. The note also records the two things that would change that and are not heuristics -- a document carrying the implementing symbol, or a configuration source answering the `${key}` the candidate already recorded -- and that the second wants its own resolver, since what it needs is the placeholder key rather than the candidate. Co-Authored-By: Claude Opus 5 (1M context) * fix(ingestion): refuse the document shapes that would forge a join Four ways a conformant AsyncAPI document could mint a Destination that connects two services which have said nothing about each other. Every one is reachable from ordinary 3.x vocabulary, not from malformed input, and each is now a countable refusal. A PARAMETERIZED ADDRESS is a pattern, not a place. Two services that publish `{env}.orders` share a template; one deploys with env=prod and the other with env=staging, and keying on the template text merges them into a single node with a publisher on one side and a subscriber on the other. This is the document-side twin of `overridable-config-default`, which argues the same thing about `${key:default}` in source. A channel declaring `parameters` is the specification's own statement that its address is a template, so the detector is a reading rather than a guess; the `{` test catches generators that template without declaring. ANY BINDINGS KEY WAS TAKEN AS A PROTOCOL. AsyncAPI allows `bindings` to be a Reference Object, so the map's own key can be `$ref` -- and passed through, that becomes half of a join key carrying no broker information at all. Two services that both reference shared bindings and both name `orders` then land on one node, defeating the broker-in-key rule that keeps `kafka orders` and `rabbit orders` apart. A broker must now be spelled like a protocol name. A BROKER CONTAINING A SPACE COLLIDES, because the node key joins with one: ("kafka orders", "x") and ("kafka", "orders x") are the same key. That was latent while every broker came from Spring's closed union. This module is the first caller to feed the shared helper text that a document wrote, which is exactly the condition under which it stops being latent, so it is closed here -- at the producer -- rather than by changing an encoding that `routeNodeKey` shares. THE ADDRESS WAS TRIMMED, while the source cascade keeps an address exactly as written so `" orders "` stays its own node. Two producers of one key held opposite whitespace policies and the document side erred toward joining. Also: fold the transport-security protocol variants (`kafka-secure`, `secure-mqtt`, `wss`, `stomps`, `https`) onto their base protocol. The `amqp`->`rabbit` argument already in this file demands it -- AsyncAPI's server vocabulary distinguishes them and its bindings vocabulary does not, so a secured cluster's own document was being read as self-contradictory. Treat a channel with no `servers` as available on all of the document's servers, which is the specification's default and was costing every single-server document its destinations. Bound the address and operation-id lengths, because `generateId` concatenates rather than hashes, and bound total operations across the run rather than only per document. Read each file through ONE handle for both the size gate and the read, as `actuator-runtime.ts` does and for the reason its comment gives (CodeQL js/file-system-race): re-resolving the path lets a swapped file bypass the cap, and the out-of-band cache this option reads is written by other tooling by definition. Open it with O_NONBLOCK: the type check that rejects a FIFO is unreachable without it, because opening a FIFO for reading blocks in open(2) until a writer appears -- found by writing the test first and watching it time out rather than fail. Count symlinked entries and walk truncation instead of dropping them in silence. A symlinked cache and a wrong path were producing identical results. Co-Authored-By: Claude Opus 5 (1M context) * fix(analyze): rebuild when documents are configured, and report what was read An AsyncAPI document is external to git freshness in exactly the way an Actuator snapshot is: replacing one moves no commit and dirties no file. The option's own README paragraph advertises an absolute cache written by other tooling, and on the second run of that workflow the already-up-to-date fast path fired, no document was ever opened, and the previous run's addresses were served as current. Measured, not reasoned: editing a document and re-running printed "Already up to date" and left the old address in the graph; with this change the same probe re-reads and the new address replaces it. So an enabled run forces a rebuild and dropping the option forces one more, to clear document-derived evidence -- the treatment `springActuatorPath` already gets, for the same reason. Only the FLAG is recorded in index metadata, not the path: Actuator retains its inputs so future scans keep excluding them, whereas a committed document is deliberately NOT excluded (it wants its real `File` node), so there is nothing to retain and recording the path would put an operator's directory layout into metadata for no consumer. That also settles a defect it would have been tempting to patch separately. A synthetic `File` node for an out-of-tree document carries a path that is in no write set and is not covered by `isGraphWideNode`, so an incremental writeback dropped the node while keeping its edges -- which then COPY against a row that was never written, and fail into an IGNORE_ERRORS retry that reports success. A forced rebuild has no incremental subgraph to get that wrong. Distinguish the two meanings of `resolution: 'specification'`. That value belongs to the address cascade and means a CODE candidate was resolved through the step-4 hook; a node minted from a document has no code site and now says `asyncapi-document`. Reusing one string would leave a query that groups by provenance unable to separate an address a document states from one a document was used to resolve, and only the second is a claim about source. Report what was read. The stats block was justified on the grounds that an operator must be able to tell a mistyped directory from a repository with no documents -- and nothing surfaced it, so the justification was aspirational. A configured path that yields nothing, or a walk that hit a bound, now warns unconditionally, as `spring-auto-configuration.ts` does for the same class of input. The phase summary carries the refusal breakdown rather than only the totals, because the unresolved fraction is the number this work is judged on and a bare count says how big the gap is without saying what would close it. Tests for the three wiring lines that were individually deletable with a green suite, following the templates already in the repository: a row in the `--watch` rejection table, the CLI-threading assertion beside the Actuator one, and the shipped-skill fragment that pins the flag in all three copies. Also pin `filePath: ''` on a spec-minted destination -- the half of the keying rule that stops a shared node becoming collateral damage of one document's next change -- and the in-repo `File` branch, which was dead-code-able. Co-Authored-By: Claude Opus 5 (1M context) * fix(ingestion): close the join-forging paths a second review round found The `$ref` exclusion added last round fixed one instance of a class and left the class open. A `bindings` map key of `x-scs-function` -- an ordinary Specification Extension, which generators emit -- still became the broker, so two unrelated services carrying one vendor annotation and one address landed on ONE node whose broker half said nothing about any broker. And `{ kafka: {}, x-internal: {} }` read as two brokers, losing a conformant document and reporting it as self-contradictory, which also made any document author a one-line saboteur of their own cross-service links. So the two readers are now separate functions with opposite defaults, and the header says why they must be. `servers[].protocol` is a FIELD DECLARED to hold a protocol: an unrecognized value there is the document's own claim and passes through, because refusing it would lose a destination the document states plainly. A `bindings` MAP KEY is not that -- the specification puts `$ref` and `x-` in the same namespace -- so a non-protocol key is the EXPECTED case and only AsyncAPI's binding vocabulary may answer. The syntactic test that was applied to both was the right rule for one of them. The walk fix from last round introduced something worse than it reported. A shared abort flag meant depth exhaustion in ONE branch terminated the whole traversal, so ten good documents beside a twelve-deep unrelated subtree were kept or lost depending on whether that subtree sorted before or after them. Truncation and budget-exhaustion are now separate: depth returns from its own branch, and only a genuinely global bound stops the walk. Rewriting `protocol.ts` dropped the whitespace check from the server-protocol path and made the node-key collision reachable again. The test written for that collision last round caught it within the minute; the comment now records that it was learned twice. Everything else measured this round: - The broker is the THIRD string that reaches a graph identifier, and it was unbounded while the header claimed there were two. A one-megabyte protocol in a document satisfying every other cap was measured producing a gigabyte of resident identifier strings, because `generateId` concatenates rather than hashes and the phase mints one id per node and per edge. - The run-wide operation budget counted ACCEPTED operations, reproducing at the run level the exact defect the per-document cap was corrected for last round: a run whose every operation is refused never decrements it. Both now count operations EXAMINED, and `operation-cap` sets `truncated` -- it is a bound that stopped the operation count, which is what that flag is documented to mean. - The channel-inherits-all-servers rule ran per operation. Hoisted: it depends only on the servers. - A subdirectory that cannot be listed is counted rather than dropped, so a mixed-permission cache cannot report a clean, complete read. - The read LOOPS, like the Actuator reader this claims to follow. A single read was never short across seven hundred probes on APFS, but POSIX permits it and FUSE mounts with `direct_io` -- the deployment this option targets -- return short counts. A document truncated at a line boundary still parses, so the failure is silent: operations vanish with `refusals: {}`. - `parameters: {}` no longer refuses a literal address; an empty container states nothing and generators emit them. - A channel that is itself a Reference Object gets its own reason instead of `no-address`, which was telling operators their documents omit addresses when the reader simply stops one hop short. - A multi-protocol document resolves from its operation's own bindings; only when those are silent does an inherited multi-protocol server set refuse, and under `ambiguous-server-default` rather than a reason that says the document contradicts itself. It does not. - HTTP and WebSocket are refused for destination minting. For a broker the topic is the namespace; for HTTP the host is, so keying on the path alone would make every service exposing `/events` one node. A `Route` already models an HTTP endpoint, with its method in the key. - The sniff window is a parse gate, not a read gate, and four kilobytes refused a good document behind a licence header. Co-Authored-By: Claude Opus 5 (1M context) * test(ingestion): pin the wiring and the reporting that were deletable Three lines could be deleted with the whole suite green, and each of them makes the feature partly or wholly inert: the forward from `run-analyze` into `PipelineOptions`, the forced rebuild while documents are configured, and the cleanup rebuild when the option is dropped. None is visible one layer up, where the CLI test asserts on a mock's arguments. One integration test closes all three. It drives the real `runFullAnalysis` against a real repository and asserts, in order: the enabled run does not take the up-to-date fast path and logs the rebuild; the destination reaches the graph, which only happens if the option is forwarded; a document edited with the tree clean and the commit unchanged is re-read; dropping the option rebuilds once and removes the document-derived evidence; and the run after that is up to date again -- the "rebuilds once" half, which rests on the metadata being written as a fresh literal rather than merged, and which nothing pinned. The document lives OUTSIDE the repository on purpose. That is the workflow the option is documented for, and it is the only one where the hazard exists: editing a tracked file dirties the tree and forces a rebuild anyway, so an in-repo fixture would pass with the freshness fix reverted. Verified by reverting both: deleting the forward fails on the empty destination list, deleting the forced rebuild fails on the missing log line. Also pinned, each because deleting the code it covers left the suite green: the `parameters` half of the templated-address refusal (its old test supplied a braced address too, so the `{` half alone satisfied it); the phase actually forwarding `symlinksSkipped`; the unconditional warning, whose whole argument is that a tally nobody can see is not a tally -- captured through the repository's own `_captureLogger`; a `.yml` document; a character device, which is the case the `isFile` check exists for and which the FIFO test does not reach; and the bound that stops a walk. Reject an empty `--asyncapi-spec` at the CLI. It resolved to the repository root and walked the whole tree, defeating this module's own rule that there is no glob-based auto-discovery -- and the HTTP entry point already rejected the identical value. Two doors onto one option must not hold different rules. Surface the flag in the MCP context resource beside `spring_actuator`. It matters more there than for its neighbour: Actuator annotates nodes the source pass already found, while document reading mints destinations and edges with no code site, and nothing said where they came from. Log the configured path relative to the repository. The same change refuses to persist that path to index metadata because it would record an operator's directory layout; holding that rule for metadata and not for logs was holding it in one place. Both test files now clean up their temporary directories. Co-Authored-By: Claude Opus 5 (1M context) * style(ingestion): apply the repository's prettier contract `quality / format` runs `npx prettier --check .`, and three files added by this branch were not formatted to it. No behaviour changes: the reader's line breaks and two test literals move, nothing else. Co-Authored-By: Claude Opus 5 (1M context) * test(ingestion): release the mini-repo handle the document test allocated `setupMiniRepo` documents that the caller owns cleanup, and every other test in this file calls `repo.cleanup()` in its `finally`. The AsyncAPI document test removed only the document directory, so each run left a temporary repository behind. The two owners are separate on purpose: the document directory is a SIBLING of the repository, placed outside the working tree so that editing it cannot dirty the tree and force a rebuild on its own. The repo's cleanup therefore does not reach it, and both calls belong in the same block. Co-Authored-By: Claude Opus 5 (1M context) * fix(ingestion): stop partial server evidence from reading as unanimous Six review findings, every one a way this reader could name a broker the document does not name. They share a shape: something is DROPPED rather than refused, the remaining evidence agrees with itself, and an operation is attributed with confidence to a broker its document never settled on. A wrong broker is half a join key, so it does not produce a missing edge -- it produces an edge to a stranger, reported as a fact. CAPPED SERVER MAPS. A channel with no `servers` inherits all of them, and that map is capped at 1,000. A document whose first thousand servers are Kafka and whose thousand-and-first is JMS read as unanimously Kafka, because unanimity was tested on the slice. Counted `server-cap` and set `truncated`, but neither stopped the attribution. The inherited path now refuses under `capped-server-default` -- checked BEFORE agreement, since a subset agrees with itself for free. ROOT SERVER REFERENCE OBJECTS. The Servers Object patterned field is `Server Object | Reference Object`, so `{ $ref: '#/components/servers/prod' }` is conformant. Reading `protocol` off the raw value dropped every one: an all-reference document had no protocol at all, and -- worse -- a MIXED set lost its disagreeing half and became unanimous. One hop is now followed, through `#/servers` and `#/components/servers`; anything else is refused under `unresolved-server-reference` rather than skipped. CHANNEL BINDINGS. Only the operation's bindings were read. A conformant channel carrying `bindings: { kafka: {} }` with no operation binding was dropped as `protocol-unknown` while the document said plainly which broker it meant, and a disagreement between the two levels was invisible. Both are read; a conflict is `protocol-disagreement`. EMPTY `servers`. "If `servers` is absent or empty, this channel MUST be available on all the servers defined in the Servers Object" -- one sentence, both cases. A zero-iteration loop returned `explicit: true`, which blocked the inherited fallback and dropped valid operations. POINTER DECODING ORDER. RFC 6901 percent-decodes the fragment BEFORE splitting on `/`. The raw token was tested for a separator first, so `#/channels/orders%2Fv1` passed a check it should have failed and then decoded into two segments -- a pointer addressing `channels.orders.v1` was read as a channel named `orders/v1`, inventing a channel the document never declared. A malformed escape is now refused rather than resolved against its undecoded text. `~1` still resolves; it is the pointer's own escape and belongs after segmentation. THE SNIFF WINDOW. A fixed window decides by where the root key sits rather than whether it is there, so every window is a false negative waiting for a longer preamble -- 4 KiB was replaced by 64 KiB for that reason and inherited the same defect. The whole text is scanned; it is already bounded and already in memory, and the gate exists to skip the PARSE, which is the expensive half. A leading UTF-8 BOM is stripped before both sniff and parse. Twelve of the fourteen new tests were run against the unfixed reader and all twelve failed; the other two are controls that must pass either way. Co-Authored-By: Claude Opus 5 (1M context) * refactor(ingestion): simplify AsyncAPI pointer and binding resolution Decode each $ref once, union binding evidence, and stop walking capped server maps whose brokers are unused on the inherit path. Co-authored-by: Cursor --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gergő Magyar Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- .claude/skills/gitnexus-cli/SKILL.md | 1 + README.md | 7 + .../skills/gitnexus-cli/SKILL.md | 1 + gitnexus/README.md | 6 + gitnexus/skills/gitnexus-cli.md | 1 + gitnexus/src/cli/analyze-options.ts | 7 + gitnexus/src/cli/analyze-watch.ts | 5 + gitnexus/src/cli/analyze.ts | 12 + gitnexus/src/cli/index.ts | 5 + .../src/core/ingestion/asyncapi/document.ts | 950 ++++++++++++++ .../src/core/ingestion/asyncapi/protocol.ts | 207 ++++ .../frameworks/spring/destinations.ts | 25 + .../pipeline-phases/spring-destinations.ts | 263 +++- gitnexus/src/core/ingestion/pipeline.ts | 14 + gitnexus/src/core/run-analyze.ts | 40 + gitnexus/src/mcp/resources.ts | 7 + gitnexus/src/server/analyze-launch.ts | 2 + gitnexus/src/server/api.ts | 9 + gitnexus/src/storage/repo-meta.ts | 17 + .../unit/analyze-worker-pool-size.test.ts | 21 + gitnexus/test/unit/asyncapi-document.test.ts | 1092 +++++++++++++++++ .../unit/incremental-orchestration.test.ts | 125 ++ .../test/unit/shipped-skills-sync.test.ts | 6 + .../unit/spring-destinations-phase.test.ts | 301 ++++- gitnexus/test/unit/watch-paths.test.ts | 5 + 25 files changed, 3125 insertions(+), 4 deletions(-) create mode 100644 gitnexus/src/core/ingestion/asyncapi/document.ts create mode 100644 gitnexus/src/core/ingestion/asyncapi/protocol.ts create mode 100644 gitnexus/test/unit/asyncapi-document.test.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index be02d92fd..09c7af0d2 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -28,6 +28,7 @@ Run from the project root. This parses all source files, builds the knowledge gr | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | | `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | +| `--asyncapi-spec ` | Read opt-in AsyncAPI 3.x documents (directory or single file) and mint `Destination` nodes from their operations. 2.x is refused, not mapped. Unsupported with `--watch`. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. diff --git a/README.md b/README.md index 0e0e2f616..a4361584a 100644 --- a/README.md +++ b/README.md @@ -450,12 +450,19 @@ gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow pa gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16, # auto-sized to the repo). 0 is rejected — there is no sequential mode. gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots +gitnexus analyze --asyncapi-spec ./docs/asyncapi # Resolve broker addresses from AsyncAPI 3.x documents gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) ``` `--spring-actuator` is explicitly opt-in and accepts either a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. It confirms matching static nodes and adds conservative runtime-only routes, beans, and property keys. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Because snapshots are external runtime state, an enabled run always rebuilds; the first later run without the option rebuilds once to remove runtime evidence. The same path can be set as `springActuator` in `.gitnexusrc`. +`--asyncapi-spec` is explicitly opt-in and accepts a directory of AsyncAPI documents or a single document; the path is resolved against the repository root, so a committed `docs/asyncapi` and an absolute cache written by something else both work. Each `operations[]` entry of an **AsyncAPI 3.x** document can contribute a `Destination` node keyed by broker and address, with `action: send` emitting `PUBLISHES_TO` and `action: receive` emitting `CONSUMES_FROM`, so a document and source code that name one address on one broker land on the same node. Edges start at the document, not at a callable — a document states that the service talks to an address, not which method does — and no address a document names is ever attached to an unresolved source site. + +An operation must name a protocol, either through its own `bindings` or through the `servers[].protocol` of the servers its channel resolves to (a channel that lists no `servers` resolves to all of them); operations that name none are refused, as are operations whose two readings name different brokers, and channels that inherit a multi-protocol server set without choosing. HTTP and WebSocket documents are refused for destination minting: there the host rather than the address names the place, and an HTTP endpoint is already modelled as a `Route`. A parameterized address — a channel declaring `parameters`, or an address containing `{` — is refused rather than keyed: two services publishing `{env}.orders` share a pattern, not a queue. AsyncAPI **2.x is refused** under its own counted reason and never mapped, because its `publish`/`subscribe` are inverted relative to 3.x `send`/`receive` and a naive mapping would reverse the async graph while leaving it connected. Every refusal is counted, and a configured path that yields nothing is reported rather than passed over in silence. + +Like Actuator snapshots, documents are external to git freshness — replacing one moves no commit and dirties no file — so an enabled run always rebuilds, and the first later run without the option rebuilds once to remove document-derived evidence. There is no glob-based auto-discovery, and the option is unsupported with `--watch`. + If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `--worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. **Embeddings node limit** — `gitnexus analyze --embeddings` generates semantic search vectors with a default 50,000-node safety cap to protect memory on large repositories: diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index be02d92fd..09c7af0d2 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -28,6 +28,7 @@ Run from the project root. This parses all source files, builds the knowledge gr | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | | `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | +| `--asyncapi-spec ` | Read opt-in AsyncAPI 3.x documents (directory or single file) and mint `Destination` nodes from their operations. 2.x is refused, not mapped. Unsupported with `--watch`. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. diff --git a/gitnexus/README.md b/gitnexus/README.md index 9abc122c5..1f0d104ac 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -349,6 +349,12 @@ anchors are deliberately omitted. Add common infrastructure fields such as `/hea `--spring-actuator` is explicitly opt-in. The path may be a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. Runtime mappings and beans confirm matching static nodes; conditions and configuration property keys enrich existing evidence, with conservative runtime-only nodes added when no match exists. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Enabled runs always rebuild because runtime snapshots are external to git freshness; omitting the option later rebuilds once to remove runtime evidence. Project config can set the same path with `springActuator` in `.gitnexusrc`. +`--asyncapi-spec` is explicitly opt-in and accepts a directory of AsyncAPI documents or a single document; the path is resolved against the repository root, so a committed `docs/asyncapi` and an absolute cache written by something else both work. Each `operations[]` entry of an **AsyncAPI 3.x** document can contribute a `Destination` node keyed by broker and address, with `action: send` emitting `PUBLISHES_TO` and `action: receive` emitting `CONSUMES_FROM`, so a document and source code that name one address on one broker land on the same node. Edges start at the document, not at a callable — a document states that the service talks to an address, not which method does — and no address a document names is ever attached to an unresolved source site. + +An operation must name a protocol, either through its own `bindings` or through the `servers[].protocol` of the servers its channel resolves to (a channel that lists no `servers` resolves to all of them); operations that name none are refused, as are operations whose two readings name different brokers, and channels that inherit a multi-protocol server set without choosing. HTTP and WebSocket documents are refused for destination minting: there the host rather than the address names the place, and an HTTP endpoint is already modelled as a `Route`. A parameterized address — a channel declaring `parameters`, or an address containing `{` — is refused rather than keyed: two services publishing `{env}.orders` share a pattern, not a queue. AsyncAPI **2.x is refused** under its own counted reason and never mapped, because its `publish`/`subscribe` are inverted relative to 3.x `send`/`receive` and a naive mapping would reverse the async graph while leaving it connected. Every refusal is counted, and a configured path that yields nothing is reported rather than passed over in silence. + +Like Actuator snapshots, documents are external to git freshness — replacing one moves no commit and dirties no file — so an enabled run always rebuilds, and the first later run without the option rebuilds once to remove document-derived evidence. There is no glob-based auto-discovery, and the option is unsupported with `--watch`. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index be02d92fd..09c7af0d2 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -28,6 +28,7 @@ Run from the project root. This parses all source files, builds the knowledge gr | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | | `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | +| `--asyncapi-spec ` | Read opt-in AsyncAPI 3.x documents (directory or single file) and mint `Destination` nodes from their operations. 2.x is refused, not mapped. Unsupported with `--watch`. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index a749589f1..646fe8494 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -129,6 +129,13 @@ export interface AnalyzeOptions { * bundle or a directory containing endpoint JSON files. Disabled by default. */ springActuator?: string; + /** + * Explicit local AsyncAPI 3.x document input. Accepts a directory of + * documents or a single document, resolved against the repository root so an + * out-of-band cache and a committed directory are equally usable. Disabled by + * default. + */ + asyncapiSpec?: string; /** OpenAI-compatible embeddings base URL (incl. /v1). Overrides GITNEXUS_EMBEDDING_URL. */ embeddingBaseUrl?: string; /** Embedding model name. Overrides GITNEXUS_EMBEDDING_MODEL. */ diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index 2e4744bc5..0dcf232d0 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -116,6 +116,11 @@ export async function resolveWatchOptions( ['--index-only', cli.indexOnly], ['--skip-git', cli.skipGit], ['--spring-actuator', cli.springActuator], + // Rejected under --watch for the same reason as --spring-actuator: the + // watcher reacts to source changes, and nothing watches an out-of-band + // document directory. Honouring the flag here would read the documents once + // and then quietly serve a stale answer for the rest of the session. + ['--asyncapi-spec', cli.asyncapiSpec], ['walCheckpointThreshold', cli.walCheckpointThreshold], ['embeddingThreads', cli.embeddingThreads], ['embeddingBatchSize', cli.embeddingBatchSize], diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index beb553e01..f2a06059e 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -1006,6 +1006,17 @@ const analyzeCommandImpl = async ( return; } + // An empty value resolves to the repository root, so `--asyncapi-spec ""` + // walks the whole tree — defeating the module's own rule that there is no + // glob-based auto-discovery, and spending the walk budget on `node_modules`. + // The HTTP entry point already rejects exactly this value; two doors onto one + // option must not hold different rules. + if (options.asyncapiSpec !== undefined && options.asyncapiSpec.trim() === '') { + cliError(' --asyncapi-spec must be a non-empty path.\n'); + process.exitCode = 1; + return; + } + if (options.embeddingDevice) { const allowed = new Set(['auto', 'cpu', 'dml', 'cuda', 'wasm']); if (!allowed.has(options.embeddingDevice)) { @@ -1375,6 +1386,7 @@ const analyzeCommandImpl = async ( // forwarded to the routes phase consumer scan. fetchWrappers: options.fetchWrappers, springActuatorPath: options.springActuator, + asyncApiSpecPath: options.asyncapiSpec, // The CLI always process.exit()s after this returns (success path at the // end of analyzeCommandImpl, error/interrupt paths via process.exit too), // so the finalize close skips the native conn/db close — it can double-free diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index d48bc65c9..6851044c0 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -163,6 +163,11 @@ program 'Import local Spring Boot Actuator JSON snapshots (mappings, beans, conditions, ' + 'configprops, env). Explicit opt-in; disabled by default.', ) + .option( + '--asyncapi-spec ', + 'Read AsyncAPI 3.x documents from this directory or file and resolve broker ' + + 'addresses from them. Explicit opt-in; disabled by default.', + ) .option('--embedding-threads ', 'Limit local ONNX embedding CPU threads') .option('--embedding-batch-size ', 'Number of nodes per embedding batch') .option('--embedding-sub-batch-size ', 'Number of chunks per embedding model call') diff --git a/gitnexus/src/core/ingestion/asyncapi/document.ts b/gitnexus/src/core/ingestion/asyncapi/document.ts new file mode 100644 index 000000000..282e57917 --- /dev/null +++ b/gitnexus/src/core/ingestion/asyncapi/document.ts @@ -0,0 +1,950 @@ +/** + * Read AsyncAPI 3.x documents off disk and normalize their operations into + * broker addresses. + * + * Deliberately OUTSIDE `frameworks/spring/`. An AsyncAPI document is a + * published artifact, not a Spring one: it is emitted by generators across + * Java, Kotlin, TypeScript, Go and Python toolchains, and it is written by hand + * as often as it is generated. The entry criterion here is therefore the + * DOCUMENT FORMAT — a root `asyncapi` key — and never the generator. Nothing in + * this module may branch on `x-generator`, on a vendor extension, or on the + * shape of an operation key: the moment it does, every service whose toolchain + * spells things differently stops being read, and the failure is silent. + * + * ── WHY THIS IS WORTH READING AT ALL ────────────────────────────────────── + * + * A `@KafkaListener(topics = "${app.topic.in}")` names a configuration key, not + * an address, and the address cascade correctly refuses to resolve it — two + * services that merely wrote the same placeholder have said nothing about each + * other. But the service's own published document states the address outright, + * fully resolved, because the generator ran with the configuration applied. + * That is a fact about the service that no amount of reading its source can + * recover. + * + * ── WHAT THIS MODULE IS AFRAID OF ───────────────────────────────────────── + * + * Everything it emits becomes half of a JOIN KEY. A destination minted here + * meets every other site in every other repository that names the same address + * on the same broker — that is the whole value, and it is the whole hazard. A + * missing destination is a visible gap; a wrong one is reported as a fact. So + * the refusals below are not defensive clutter: each one is a case where the + * document says something that LOOKS like an address and is not one, and where + * accepting it would connect two services that have said nothing about each + * other. The taxonomy is closed and countable for the same reason the source + * cascade's is — a feature judged on its unresolved fraction needs the fraction + * broken down by cause, or nobody can tell it what to go and fix. + * + * ── VERSION 2.x IS REFUSED, NOT MAPPED ──────────────────────────────────── + * + * AsyncAPI 2.x describes a channel from the READER's point of view: `publish` + * means "you may publish here", so the documenting application RECEIVES, and + * `subscribe` means the application SENDS. Version 3.0 renamed these to the + * application's own `receive` / `send`. Mapping 2.x naively therefore reverses + * every direction in the async graph — and reverses it INVISIBLY, because both + * roles still exist, every edge is still emitted, and the graph stays + * connected. Nothing fails; the arrows simply point the wrong way. + * + * The inversion is one line to write and impossible to test against a real + * corpus we do not have, and the 2.x wording confused implementers badly enough + * that some generators emitted it backwards. So 2.x is refused under its own + * countable reason instead. A silent skip would be indistinguishable from "this + * service publishes no document", which is the one thing the count has to be + * able to tell us: if the refusal tally shows 2.x documents in the field, the + * inversion earns its way in with evidence behind it. + */ + +import { createRequire } from 'node:module'; +import fs from 'node:fs/promises'; +import { constants as fsConstants } from 'node:fs'; +import path from 'node:path'; +import { brokerForBindingKey, brokerForProtocol, isNonDestinationBroker } from './protocol.js'; + +// `js-yaml` is CJS; the rest of this repository reaches it the same way +// (`pipeline-phases/spring-config.ts`, `import-resolvers/node-workspace-packages.ts`). +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +/** + * A published document is data, not code, so it is parsed under the JSON + * schema — the same choice `core/group/config-parser.ts` makes for `group.yaml`. + * No custom tags, no timestamps, no `yes`/`no` booleans: an address is whatever + * the document literally spells, and nothing may be coerced into another type + * on the way in. + */ +const DOCUMENT_SCHEMA = yaml.JSON_SCHEMA; + +/** Generous for a specification, small enough that a mistake is caught. */ +const MAX_DOCUMENT_BYTES = 8 * 1024 * 1024; +/** Bounded so one pathological document cannot dominate a run. Counted against + * operations EXAMINED, not accepted: a document with a hundred thousand + * refused operations costs the same walk as one with a hundred thousand good + * ones, and a cap that only counts successes does not bound the work. */ +const MAX_OPERATIONS_PER_DOCUMENT = 5_000; +/** The same bound across the whole run, and counted the same way — EXAMINED, + * not accepted. Counting successes here reproduced the very defect the + * per-document cap was corrected for: a run whose every operation was refused + * never decremented the budget, so all two thousand documents were processed + * in full and the result still reported `truncated: false`. */ +const MAX_TOTAL_OPERATIONS = 50_000; +/** Bounded so a mis-aimed path (a whole repository, `/`) cannot walk forever. */ +const MAX_DOCUMENTS = 2_000; +/** Directory entries VISITED, not documents accepted. The document cap alone + * bounds nothing on a tree that contains no documents. */ +const MAX_WALK_ENTRIES = 100_000; +const MAX_DIRECTORY_DEPTH = 8; +/** Servers per document. The channel-inherits-all-servers rule reads this map, + * and YAML aliases make a server about sixteen bytes, so an in-cap document + * can declare hundreds of thousands of them. */ +const MAX_SERVERS_PER_DOCUMENT = 1_000; +/** + * A leading byte-order mark, stripped before the file is sniffed or parsed. + * + * An editor that saves UTF-8 with a BOM puts one code point in front of the + * root key, which is enough to make the sniff miss and refuse a perfectly good + * document as `not-a-document`. + */ +const BOM = '\uFEFF'; +/** + * An address and an operation id both end up inside graph identifiers, and + * `generateId` CONCATENATES rather than hashes (`lib/utils.ts`), so an + * identifier is exactly as long as the text it was built from. One document + * under every other cap — a multi-megabyte address plus five thousand + * operations naming it — therefore mints five thousand multi-megabyte edge ids, + * each flattened into a string key by the graph's `Map`. + * + * The BROKER is the third such string and is bounded in `protocol.ts`; the + * count matters because an earlier version of this comment said "the two + * strings that reach an id", left the third unbounded, and a one-megabyte + * protocol was measured turning a one-megabyte document into a gigabyte of + * resident identifiers. + */ +const MAX_ADDRESS_LENGTH = 2_048; +const MAX_OPERATION_ID_LENGTH = 512; + +const DOCUMENT_EXTENSIONS: ReadonlySet = new Set(['.yaml', '.yml', '.json']); + +/** + * Why a document, or one operation inside it, produced no address. + * + * A CLOSED, COUNTABLE set, and deliberately NOT `SpringDestinationRefusal`. + * That union is documented as the reasons a *source-level candidate* produced + * no address, and it is the denominator of the unresolved fraction the address + * work is judged on. Folding document-level failures into it would silently + * change what that number means — a repository whose specification directory + * was mistyped would report a worse SOURCE, which is the opposite of the truth. + * + * Members are split wherever two causes are different FACTS about the input. + * A tally whose member says "the document contradicts itself" when the document + * is merely multi-protocol sends an operator to fix the wrong thing, and this + * tally is the number the whole feature is judged on. + */ +export type AsyncApiRefusal = + /** The file parsed but has no root `asyncapi` key: not a document at all. */ + | 'not-a-document' + /** Root `asyncapi: 2.x`. See the header — refused, never mapped. */ + | 'asyncapi-2-unsupported' + /** A root `asyncapi` key naming a version this module does not read. */ + | 'unsupported-version' + /** Malformed YAML/JSON, or a root that is not an object. */ + | 'unparsable' + /** The file could not be read, or is not a regular file (a FIFO, a device). */ + | 'unreadable' + /** A subdirectory could not be listed. Counted rather than skipped: under a + * mixed-permission cache half the documents can be invisible while the run + * otherwise reports a clean, complete read. */ + | 'directory-unreadable' + /** Larger than {@link MAX_DOCUMENT_BYTES}. */ + | 'oversized' + /** The document held more operations than one run will examine. */ + | 'operation-cap' + /** The run as a whole reached {@link MAX_TOTAL_OPERATIONS}. */ + | 'total-operation-cap' + /** The document declares more servers than the channel-inheritance rule will + * read. */ + | 'server-cap' + /** The walk hit a bound before it finished, so the document set is a floor + * rather than the whole of what the configured path holds. A truncated read + * that reported nothing would be indistinguishable from a complete one. */ + | 'walk-truncated' + /** `operations[].channel.$ref` is absent or not a local channel pointer. */ + | 'no-channel-reference' + /** The `$ref` resolved to no channel in this document. */ + | 'channel-not-found' + /** The channel entry is itself a Reference Object, which this module does not + * follow. Distinct from `no-address` on purpose: a `$ref`-ed channel HAS an + * address, somewhere this reader did not look, and filing it under + * `no-address` tells an operator their documents omit addresses when the + * real answer is that the reader stops one hop short. */ + | 'unresolved-channel-reference' + /** The channel names no `address`, so there is nothing to key on. */ + | 'no-address' + /** + * The address is a TEMPLATE, not an address: the channel declares non-empty + * `parameters`, or the address carries a `{…}` placeholder. + * + * This is the document-side twin of the source cascade's + * `overridable-config-default`, and it exists for the identical reason. Two + * services that both publish `{env}.orders` have named a pattern they share, + * not a queue they share — one deploys with `env=prod` and the other with + * `env=staging`, and keying on the template text merges them into a single + * node with a publisher on one side and a subscriber on the other. That is a + * false connection built entirely from conformant AsyncAPI: `parameters` and + * `{param}` are core 3.x vocabulary, not a vendor quirk. + */ + | 'templated-address' + /** Longer than {@link MAX_ADDRESS_LENGTH}. */ + | 'address-too-long' + /** Longer than {@link MAX_OPERATION_ID_LENGTH}. */ + | 'operation-id-too-long' + /** `action` is neither `send` nor `receive`. */ + | 'unrecognized-action' + /** Neither the operation's bindings nor the servers its channel resolves to + * name a protocol. Silence about the broker is not a claim about it, but a + * `Destination` cannot be keyed without one. */ + | 'protocol-unknown' + /** The operation's OWN two statements about its broker — its bindings and the + * servers its channel explicitly lists — name different brokers. The + * document contradicts itself, and a destination keyed on the wrong broker + * joins a stranger. */ + | 'protocol-disagreement' + /** The channel lists no `servers`, so it inherits all of them, and they do + * not agree on one broker. The document does NOT contradict itself here — + * it is simply multi-protocol and this channel did not choose — which is why + * this is not `protocol-disagreement`. */ + | 'ambiguous-server-default' + /** + * The channel inherits the document's servers, but that map was CAPPED at + * {@link MAX_SERVERS_PER_DOCUMENT}, so the brokers read are a subset. + * + * Distinct from `ambiguous-server-default`, and the distinction is the whole + * point: unanimity across a subset is not unanimity. A document whose first + * thousand servers are Kafka and whose thousand-and-first is JMS reads as + * unanimously Kafka, and every operation inheriting it would be attributed to + * a broker the complete set does not agree on. + */ + | 'capped-server-default' + /** + * A server this operation depends on is a Reference Object this reader could + * not resolve — a pointer outside `#/servers` and `#/components/servers`, a + * name that is absent, or a reference to another reference. + * + * Refused rather than skipped. Skipping one server of several silently + * narrows the evidence, and a narrowed set is what makes a mixed document + * look like it agrees with itself. + */ + | 'unresolved-server-reference' + /** The broker is HTTP or WebSocket, where the host rather than the address is + * the namespace. See `isNonDestinationBroker`. */ + | 'not-a-destination-protocol'; + +export interface AsyncApiOperation { + /** Absolute path of the document this operation came from. */ + readonly documentPath: string; + /** The `operations` map key, kept for provenance and carried into the edge + * `reason` so a reader can find the operation the edge came from. */ + readonly operationId: string; + readonly action: 'send' | 'receive'; + readonly address: string; + /** Normalized broker — the first half of the `Destination` key. */ + readonly broker: string; +} + +export interface AsyncApiReadResult { + readonly operations: readonly AsyncApiOperation[]; + /** Files considered — every candidate extension under the configured path. */ + readonly documentsScanned: number; + /** Files that parsed as an AsyncAPI 3.x document and yielded an operation. */ + readonly documentsAccepted: number; + /** Entries skipped because they were symbolic links. Not a refusal — the skip + * is deliberate — but counted, because a cache written by other tooling is + * very often a symlink farm, and an operator whose whole cache was skipped + * would otherwise see a result identical to a wrong path. */ + readonly symlinksSkipped: number; + /** True when a bound stopped the walk or the operation count, so every number + * here is a floor rather than a total. */ + readonly truncated: boolean; + /** Every refusal, document-level and operation-level, by reason. */ + readonly refusals: Readonly>>; +} + +interface Tally { + count(reason: AsyncApiRefusal): void; +} + +function makeTally(sink: Partial>): Tally { + return { + count: (reason) => { + sink[reason] = (sink[reason] ?? 0) + 1; + }, + }; +} + +/** + * Own-property read that cannot be answered by the prototype chain. + * + * A document is untrusted input and its keys are attacker-chosen in the general + * case. `channels['constructor']` misses because {@link asRecord} rejects a + * function — but `channels['__proto__']` would otherwise resolve to + * `Object.prototype`, which IS an object and would sail through as an empty + * channel. This guard, not the type test, is what stops that one. + */ +function own(container: unknown, key: string): unknown { + if (typeof container !== 'object' || container === null) return undefined; + if (!Object.prototype.hasOwnProperty.call(container, key)) return undefined; + return (container as Record)[key]; +} + +function asRecord(value: unknown): Record | undefined { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined; + return value as Record; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** URI-fragment percent-decode. Malformed `%` sequences refuse the pointer. */ +function decodeFragment(ref: string): string | undefined { + try { + return decodeURIComponent(ref); + } catch (err) { + // Malformed percent-escapes throw URIError. Anything else is a real bug. + if (!(err instanceof URIError)) throw err; + return undefined; + } +} + +/** RFC 6901's own escapes, `~1` before `~0` — a literal `~1` produced by + * decoding `~01` would otherwise be mistaken for a slash. */ +function unescapePointerToken(token: string): string { + return token.split('~1').join('/').split('~0').join('~'); +} + +/** + * Trailing name of `` on an already-decoded fragment. + * + * DECODE, THEN SEGMENT. RFC 6901 percent-decodes the URI fragment first; only + * then is the result split on `/`. Testing the raw text for a separator lets + * `#/channels/orders%2Fv1` through as one segment and then decode it into two, + * inventing a channel named `orders/v1`. A real slash in a name is `~1`. + */ +function nameAfterPrefix(decoded: string, prefix: string): string | undefined { + if (!decoded.startsWith(prefix)) return undefined; + const token = decoded.slice(prefix.length); + if (token === '' || token.includes('/')) return undefined; + return unescapePointerToken(token); +} + +function pointerName(ref: string, prefix: string): string | undefined { + const decoded = decodeFragment(ref); + if (decoded === undefined) return undefined; + return nameAfterPrefix(decoded, prefix); +} + +/** + * Distinct brokers named by a bindings object's own keys. + * + * Routed through `brokerForBindingKey`, which answers only for AsyncAPI's + * binding vocabulary — `$ref` and `x-` extensions share this namespace + * legitimately and are not brokers. See `protocol.ts` for why this differs from + * the pass-through applied to `servers[].protocol`. + */ +function brokersFromBindings(bindings: unknown): Set { + const out = new Set(); + const record = asRecord(bindings); + if (record === undefined) return out; + for (const key of Object.keys(record)) { + const broker = brokerForBindingKey(key); + if (broker !== undefined) out.add(broker); + } + return out; +} + +/** + * The broker one Servers Object entry names, following at most one local `$ref`. + * + * The Servers Object's patterned field is `Server Object | Reference Object`, + * so an entry may legitimately be `{ $ref: '#/components/servers/prod' }`. + * Reading `protocol` off the raw value drops every one of those, and a dropped + * server is not neutral here: in a mixed set it removes the disagreeing half + * and makes partial evidence look unanimous, which is exactly how a confident + * WRONG broker gets attributed. + * + * `unresolved` is reported rather than swallowed so the caller can refuse the + * attribution instead of answering from the servers it happened to understand. + * A reference to a reference counts as unresolved too: one hop covers every + * document shape seen in practice, and chasing a chain over untrusted input + * would need a cycle guard before it were safe at all. + */ +function serverBroker( + entry: unknown, + root: Record, +): { broker: string | undefined; unresolved: boolean } { + const ref = asString(own(entry, '$ref')); + if (ref === undefined) { + return { broker: brokerForProtocol(asString(own(entry, 'protocol'))), unresolved: false }; + } + const target = resolveLocalServerRef(ref, root); + if (target === undefined || own(target, '$ref') !== undefined) { + return { broker: undefined, unresolved: true }; + } + return { broker: brokerForProtocol(asString(own(target, 'protocol'))), unresolved: false }; +} + +/** `#/servers/` or `#/components/servers/` → that Server Object. */ +function resolveLocalServerRef( + ref: string, + root: Record, +): Record | undefined { + const decoded = decodeFragment(ref); + if (decoded === undefined) return undefined; + const direct = nameAfterPrefix(decoded, '#/servers/'); + if (direct !== undefined) return asRecord(own(asRecord(own(root, 'servers')), direct)); + const inComponents = nameAfterPrefix(decoded, '#/components/servers/'); + if (inComponents === undefined) return undefined; + const components = asRecord(own(asRecord(own(root, 'components')), 'servers')); + return asRecord(own(components, inComponents)); +} + +/** + * Brokers of the servers a channel names explicitly. + * + * An EMPTY array is not an explicit choice. The specification defines the two + * cases identically — "If `servers` is absent or empty, this channel MUST be + * available on all the servers defined in the Servers Object" — so reporting + * `explicit: true` after a zero-iteration loop blocks the inherited fallback + * and drops a perfectly valid operation as `protocol-unknown`. + * + * A channel's `servers` MUST hold Reference Objects — the specification says so + * in as many words, and forbids Server Objects there by name — so an entry that + * is not a resolvable local reference is counted `unresolved` rather than read. + */ +function brokersFromChannelRefs( + channel: Record, + root: Record, +): { brokers: Set; explicit: boolean; unresolved: boolean } { + const out = new Set(); + const refs = own(channel, 'servers'); + if (!Array.isArray(refs) || refs.length === 0) { + return { brokers: out, explicit: false, unresolved: false }; + } + const servers = asRecord(own(root, 'servers')); + let unresolved = false; + for (const entry of refs) { + const ref = asString(own(entry, '$ref')); + const name = ref === undefined ? undefined : pointerName(ref, '#/servers/'); + const target = name === undefined ? undefined : own(servers, name); + if (target === undefined) { + unresolved = true; + continue; + } + const resolved = serverBroker(target, root); + if (resolved.unresolved) { + unresolved = true; + continue; + } + if (resolved.broker !== undefined) out.add(resolved.broker); + } + return { brokers: out, explicit: true, unresolved }; +} + +/** + * Every broker the document's servers name, computed ONCE per document. + * + * A channel that names no `servers` is available on all of them — the + * specification's own default, not an inference. Computing it per operation was + * quadratic in `servers × operations`, which an in-cap document can drive to + * minutes. + */ +function brokersOfAllServers(root: Record): { + brokers: Set; + capped: boolean; + unresolved: boolean; +} { + const out = new Set(); + const servers = asRecord(own(root, 'servers')); + if (servers === undefined) return { brokers: out, capped: false, unresolved: false }; + let seen = 0; + for (const name in servers) { + if (!Object.prototype.hasOwnProperty.call(servers, name)) continue; + seen += 1; + if (seen > MAX_SERVERS_PER_DOCUMENT) { + // Inherited resolution refuses a capped map before asking it to agree + // with itself, so the brokers of the first thousand entries are unused. + return { brokers: out, capped: true, unresolved: false }; + } + } + let unresolved = false; + for (const name in servers) { + if (!Object.prototype.hasOwnProperty.call(servers, name)) continue; + const resolved = serverBroker(own(servers, name), root); + if (resolved.unresolved) unresolved = true; + else if (resolved.broker !== undefined) out.add(resolved.broker); + } + return { brokers: out, capped: false, unresolved }; +} + +/** + * Root `asyncapi` version → readable, refused, or not a document at all. + * + * Compared on the MAJOR component only. A 3.1 document adds fields this module + * does not read and changes none it does; refusing it would lose real + * destinations over a minor-version digit. + */ +function classifyVersion(raw: Record): 'read' | AsyncApiRefusal { + const declared = asString(own(raw, 'asyncapi'))?.trim(); + if (declared === undefined || declared === '') return 'not-a-document'; + const major = declared.split('.')[0]; + if (major === '3') return 'read'; + if (major === '2') return 'asyncapi-2-unsupported'; + return 'unsupported-version'; +} + +export interface NormalizedDocument { + operations: AsyncApiOperation[]; + refusals: Partial>; + /** Operations EXAMINED, which is what the caps count. */ + examined: number; + /** A bound stopped this document short. */ + truncated: boolean; +} + +/** + * Normalize one parsed document. Pure — no filesystem, so the whole refusal + * surface is testable from inline document literals. + * + * `budget` is the number of operations the RUN may still examine. + */ +export function normalizeAsyncApiDocument( + parsed: unknown, + documentPath: string, + budget: number = MAX_TOTAL_OPERATIONS, +): NormalizedDocument { + const refusals: Partial> = {}; + const tally = makeTally(refusals); + const operations: AsyncApiOperation[] = []; + let examined = 0; + let truncated = false; + + const raw = asRecord(parsed); + if (raw === undefined) { + tally.count('unparsable'); + return { operations, refusals, examined, truncated }; + } + + const verdict = classifyVersion(raw); + if (verdict !== 'read') { + tally.count(verdict); + return { operations, refusals, examined, truncated }; + } + + const channels = asRecord(own(raw, 'channels')); + const operationsRaw = asRecord(own(raw, 'operations')); + if (operationsRaw === undefined) return { operations, refusals, examined, truncated }; + + const allServers = brokersOfAllServers(raw); + if (allServers.capped) { + tally.count('server-cap'); + truncated = true; + } + + for (const operationId of Object.keys(operationsRaw)) { + if (examined >= MAX_OPERATIONS_PER_DOCUMENT) { + tally.count('operation-cap'); + truncated = true; + break; + } + if (examined >= budget) { + tally.count('total-operation-cap'); + truncated = true; + break; + } + examined += 1; + + const operation = asRecord(own(operationsRaw, operationId)); + if (operation === undefined) { + tally.count('unparsable'); + continue; + } + + if (operationId.length > MAX_OPERATION_ID_LENGTH) { + tally.count('operation-id-too-long'); + continue; + } + + const action = asString(own(operation, 'action'))?.trim().toLowerCase(); + if (action !== 'send' && action !== 'receive') { + tally.count('unrecognized-action'); + continue; + } + + const ref = asString(own(own(operation, 'channel'), '$ref')); + const channelName = ref === undefined ? undefined : pointerName(ref, '#/channels/'); + if (channelName === undefined) { + tally.count('no-channel-reference'); + continue; + } + const channel = asRecord(own(channels, channelName)); + if (channel === undefined) { + tally.count('channel-not-found'); + continue; + } + if (own(channel, '$ref') !== undefined && own(channel, 'address') === undefined) { + tally.count('unresolved-channel-reference'); + continue; + } + + // The `address` field, not the channel KEY. A generator is free to key a + // channel by anything unique; only `address` is defined as the thing the + // broker is addressed by, and keying a node on a document-local map key + // would join two services that merely organized their documents alike. + // + // NOT TRIMMED, deliberately. The source cascade keeps an address exactly as + // written — `" orders "` is its own node and does not join `"orders"` — on + // the grounds that a missing connection beats a false one. Two producers of + // one key must not hold opposite whitespace policies, and of the two + // available answers this is the one that errs away from joining. + const address = asString(own(channel, 'address')); + if (address === undefined || address.trim() === '') { + tally.count('no-address'); + continue; + } + if (address.length > MAX_ADDRESS_LENGTH) { + tally.count('address-too-long'); + continue; + } + // A non-empty `parameters` map is the specification's own statement that + // the address is a template. An EMPTY one states nothing — generators emit + // empty containers routinely — so it must not refuse a literal address; the + // `{` test below covers documents that template without declaring. + const parameters = asRecord(own(channel, 'parameters')); + if ((parameters !== undefined && Object.keys(parameters).length > 0) || address.includes('{')) { + tally.count('templated-address'); + continue; + } + + // BINDINGS FIRST. They are the operation's own statement about its broker; + // the servers are the channel's. Unioning every server before consulting + // the bindings made a document that declares both a REST server and a Kafka + // server lose every operation it states, filed under a reason that says the + // document contradicts itself — when the contradiction was manufactured + // here by asking a question the operation had already answered. + // + // The CHANNEL's bindings count as well. They are a statement about the same + // operation made one level up, and a conformant document may carry only + // those — `channels: { orders: { bindings: { kafka: {} } } }` with no + // operation binding and no usable server protocol was dropped as + // `protocol-unknown` while the document had said plainly which broker it + // meant. Where both levels speak and disagree, the document contradicts + // itself and neither answer may be used. + const fromBindings = brokersFromBindings(own(operation, 'bindings')); + for (const broker of brokersFromBindings(own(channel, 'bindings'))) { + fromBindings.add(broker); + } + if (fromBindings.size > 1) { + tally.count('protocol-disagreement'); + continue; + } + const bindingBroker = [...fromBindings][0]; + + const explicitServers = brokersFromChannelRefs(channel, raw); + let broker: string | undefined; + if (bindingBroker !== undefined) { + // Cross-check only against servers the channel named itself, and only + // when they are unanimous. An inherited multi-protocol server set is not + // a claim about THIS operation. + const explicitBroker = + explicitServers.brokers.size === 1 ? [...explicitServers.brokers][0] : undefined; + if (explicitBroker !== undefined && explicitBroker !== bindingBroker) { + tally.count('protocol-disagreement'); + continue; + } + broker = bindingBroker; + } else if (explicitServers.explicit) { + if (explicitServers.unresolved) { + tally.count('unresolved-server-reference'); + continue; + } + if (explicitServers.brokers.size > 1) { + tally.count('protocol-disagreement'); + continue; + } + broker = [...explicitServers.brokers][0]; + } else { + // Order matters: an INCOMPLETE set must be refused before it is asked + // whether it agrees, because a subset agrees with itself for free. + if (allServers.capped) { + tally.count('capped-server-default'); + continue; + } + if (allServers.unresolved) { + tally.count('unresolved-server-reference'); + continue; + } + if (allServers.brokers.size > 1) { + tally.count('ambiguous-server-default'); + continue; + } + broker = [...allServers.brokers][0]; + } + + if (broker === undefined) { + tally.count('protocol-unknown'); + continue; + } + if (isNonDestinationBroker(broker)) { + tally.count('not-a-destination-protocol'); + continue; + } + + operations.push({ documentPath, operationId, action, address, broker }); + } + + return { operations, refusals, examined, truncated }; +} + +/** + * Cheap pre-parse gate: does this file even claim to be an AsyncAPI document? + * + * Scans the WHOLE text, which is already bounded by {@link MAX_DOCUMENT_BYTES} + * and already in memory. A fixed window is the wrong shape of bound here: it + * decides the answer by where the key happens to sit rather than by whether the + * key is there, so any window is a false negative waiting for a file with a + * longer preamble. Sixty-four kilobytes replaced four for exactly that reason + * and inherited exactly that defect — a licence header, a `$schema` block and a + * long `info.description` clear it easily. The gate exists to skip the YAML + * PARSE, which is the expensive half; a linear scan of the same bytes is not. + */ +function looksLikeDocument(text: string): boolean { + if (!text.includes('asyncapi')) return false; + return /(^|[\s{,"'])["']?asyncapi["']?\s*:/m.test(text); +} + +interface WalkResult { + files: string[]; + symlinksSkipped: number; + truncated: boolean; + unreadableDirectories: number; +} + +async function collectCandidateFiles(root: string): Promise { + const files: string[] = []; + let symlinksSkipped = 0; + let unreadableDirectories = 0; + let visited = 0; + let truncated = false; + // Distinct from `truncated`: a GLOBAL budget is exhausted and no further work + // is useful, whereas depth exhaustion in one branch says nothing about its + // siblings. Conflating them made a single over-deep subdirectory discard + // every remaining document in the walk, with the outcome decided by + // alphabetical ordering — a strictly worse failure than the one the + // truncation reporting was added to fix. + let exhausted = false; + + const walk = async (dir: string, depth: number): Promise => { + if (exhausted) return; + if (depth > MAX_DIRECTORY_DEPTH) { + truncated = true; + return; + } + let entries: import('node:fs').Dirent[]; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + unreadableDirectories += 1; + truncated = true; + return; + } + // Sorted so the operation order a run produces is a function of the tree, + // not of the order the filesystem happened to hand entries back. + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const entry of entries) { + if (exhausted) return; + visited += 1; + if (visited > MAX_WALK_ENTRIES || files.length >= MAX_DOCUMENTS) { + truncated = true; + exhausted = true; + return; + } + const full = path.join(dir, entry.name); + // `withFileTypes` reports a symlink as neither file nor directory, so + // links are skipped without ever being followed — a configured directory + // must not become a route out of itself. Counted rather than dropped in + // silence: a symlinked cache would otherwise look exactly like a wrong + // path. + if (entry.isSymbolicLink()) { + symlinksSkipped += 1; + } else if (entry.isDirectory()) { + await walk(full, depth + 1); + } else if ( + entry.isFile() && + DOCUMENT_EXTENSIONS.has(path.extname(entry.name).toLowerCase()) + ) { + files.push(full); + } + } + }; + + await walk(root, 0); + return { files, symlinksSkipped, truncated, unreadableDirectories }; +} + +/** + * Read one candidate file under a hard byte ceiling. + * + * Modelled on `frameworks/spring/actuator-runtime.ts`'s `readPayloadFile`, and + * for the reason its comment gives: the size gate and the read share ONE handle + * so both observe the same inode. Checking `fs.stat(path)` and then re-resolving + * that path in `fs.readFile` lets whatever writes the directory swap the file + * between the two calls, which makes the cap advisory (CodeQL + * js/file-system-race). The out-of-band cache this option exists to read is + * written by other tooling by definition, so the race is the normal condition + * here rather than an exotic one. + * + * The read LOOPS, like its model. POSIX permits a short read on a regular file, + * and a single read was measured never short across seven hundred reads on + * APFS — but the deployments this option targets put the cache on NFS, SMB or a + * FUSE mount, and FUSE filesystems using `direct_io` do return short counts. + * The consequence of one short read is silent: a document truncated at a line + * boundary still parses, so operations vanish with `refusals: {}` and + * `truncated: false`, indistinguishable from a document that had fewer. + * + * The `isFile` test on the same handle is what the path-based version could not + * do at all. Without it the single-file configuration accepts anything `stat` + * follows: a character device reports size 0 and then streams until Node throws + * at two gigabytes, and a FIFO never returns at all. + * + * `O_NONBLOCK` is what makes that test reachable, and it was not obvious — it + * was found by writing the FIFO test and watching it TIME OUT rather than fail. + * Opening a FIFO for reading blocks in `open(2)` until some writer opens the + * other end, so a type check performed after the open never runs: the analyze + * hangs there, holding its repository lock, with no error to report. The flag + * makes the open return immediately for a FIFO and is a no-op for the regular + * files this actually wants (measured: identical byte count, digest and timing + * with and without it), which is why it costs nothing to keep. + */ +async function readBoundedFile(file: string): Promise { + let handle: import('node:fs/promises').FileHandle | undefined; + try { + // `O_NONBLOCK` is absent on some platforms; falling back to a plain + // read-only open there keeps behaviour identical for regular files. + const nonBlocking = (fsConstants.O_NONBLOCK ?? 0) | fsConstants.O_RDONLY; + handle = await fs.open(file, nonBlocking); + const stat = await handle.stat(); + if (!stat.isFile()) return 'unreadable'; + if (stat.size > MAX_DOCUMENT_BYTES) return 'oversized'; + const buffer = Buffer.alloc(MAX_DOCUMENT_BYTES + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const chunk = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (chunk.bytesRead === 0) break; + bytesRead += chunk.bytesRead; + } + if (bytesRead > MAX_DOCUMENT_BYTES) return 'oversized'; + return buffer.subarray(0, bytesRead).toString('utf-8'); + } catch { + return 'unreadable'; + } finally { + await handle?.close().catch(() => {}); + } +} + +/** + * Read every AsyncAPI 3.x document under an explicitly configured path. + * + * `configuredPath` is resolved against the repository root, so an absolute path + * to a cache populated out of band and a repo-relative directory of committed + * documents are both natural — the same shape `springActuatorPath` offers for + * Actuator snapshots. The READ is wider than that neighbour's, and the + * difference is worth stating rather than glossed as "the same contract": the + * Actuator loader probes five fixed filenames in one directory, while this + * walks recursively under the caps above and opens every candidate it finds. + * + * There is deliberately NO glob-based auto-discovery. Scanning a repository for + * anything that parses as a document would make every existing index grow nodes + * on its next run with nobody having asked for it. + */ +export async function readAsyncApiDocuments( + repoPath: string, + configuredPath: string, +): Promise { + const refusals: Partial> = {}; + const tally = makeTally(refusals); + const operations: AsyncApiOperation[] = []; + let documentsScanned = 0; + let documentsAccepted = 0; + let symlinksSkipped = 0; + let truncated = false; + let examinedTotal = 0; + + const root = path.resolve(repoPath, configuredPath); + let files: string[]; + try { + const stat = await fs.stat(root); + if (stat.isDirectory()) { + const walked = await collectCandidateFiles(root); + files = walked.files; + symlinksSkipped = walked.symlinksSkipped; + truncated = walked.truncated; + for (let i = 0; i < walked.unreadableDirectories; i += 1) tally.count('directory-unreadable'); + if (truncated && walked.unreadableDirectories === 0) tally.count('walk-truncated'); + } else { + files = [root]; + } + } catch { + tally.count('unreadable'); + return { + operations, + documentsScanned, + documentsAccepted, + symlinksSkipped, + truncated, + refusals, + }; + } + + for (const file of files) { + documentsScanned += 1; + const content = await readBoundedFile(file); + if (content === 'oversized' || content === 'unreadable') { + tally.count(content); + continue; + } + + const text = content.startsWith(BOM) ? content.slice(BOM.length) : content; + + // Sniff before parsing. A configured directory may hold hundreds of + // unrelated YAML files, and parsing each one to discover it is not a + // document is the difference between a bounded cost and a per-file one. + if (!looksLikeDocument(text)) { + tally.count('not-a-document'); + continue; + } + + let parsed: unknown; + try { + parsed = yaml.load(text, { schema: DOCUMENT_SCHEMA }); + } catch { + tally.count('unparsable'); + continue; + } + + const remaining = MAX_TOTAL_OPERATIONS - examinedTotal; + if (remaining <= 0) { + tally.count('total-operation-cap'); + truncated = true; + break; + } + + const result = normalizeAsyncApiDocument(parsed, file, remaining); + examinedTotal += result.examined; + if (result.truncated) truncated = true; + for (const [reason, count] of Object.entries(result.refusals)) { + refusals[reason as AsyncApiRefusal] = (refusals[reason as AsyncApiRefusal] ?? 0) + count; + } + if (result.operations.length > 0) documentsAccepted += 1; + operations.push(...result.operations); + } + + return { operations, documentsScanned, documentsAccepted, symlinksSkipped, truncated, refusals }; +} diff --git a/gitnexus/src/core/ingestion/asyncapi/protocol.ts b/gitnexus/src/core/ingestion/asyncapi/protocol.ts new file mode 100644 index 000000000..dd02df2dd --- /dev/null +++ b/gitnexus/src/core/ingestion/asyncapi/protocol.ts @@ -0,0 +1,207 @@ +/** + * AsyncAPI protocol name → broker identity, for `destinationNodeKey`. + * + * Deliberately OUTSIDE `frameworks/spring/`, like `destination-key.ts` and for + * the same reason: an AsyncAPI document is not a Spring artifact, and the + * broker it names has to be mintable by anything that reads one. + * + * ── TWO READERS, TWO RULES, AND WHY THEY ARE NOT THE SAME RULE ──────────── + * + * A document states its protocol in two places, and they have opposite + * defaults: + * + * `servers[].protocol` is a FIELD DECLARED TO HOLD A PROTOCOL. Whatever it + * contains is the document's claim about its broker, including a protocol + * this codebase has never heard of. {@link brokerForProtocol} therefore + * passes an unrecognized value through as its own literal — an `mqtt` or + * `nats` channel mints `mqtt
` and joins any other site that says + * the same thing, instead of being dropped for the sake of a closed union it + * was never going to fit. Refusing it would lose a destination the document + * states plainly, to protect against a collision that cannot happen: an + * unmapped protocol keys on its own name, so it can only meet a site that + * named the same protocol. + * + * A `bindings` MAP KEY is not that. The map is keyed by protocol name BY + * CONVENTION, and the specification puts other things in the same namespace: + * `$ref` when the bindings are a Reference Object, and `x-` Specification + * Extensions, which generators emit routinely. Here a non-protocol key is the + * EXPECTED case, not the exotic one, so {@link brokerForBindingKey} answers + * only for names it recognizes. + * + * That asymmetry was learned the expensive way. An earlier version applied one + * syntactic test to both and excluded only `$`-prefixed tokens; a document + * carrying `bindings: { x-scs-function: … }` then minted + * `Destination(broker='x-scs-function')`, so two unrelated services sharing a + * vendor annotation and an address landed on ONE node with a broker half that + * carried no broker information at all. Worse, `{ kafka: {}, x-internal: {} }` + * read as two brokers and refused a conformant document as self-contradictory + * — which also makes any writer of a document a one-line saboteur of its own + * cross-service links. A list that must be extended when AsyncAPI adds a + * binding is the smaller cost. + * + * ── WHY THE ALIASES EARN THEIR ROWS ─────────────────────────────────────── + * + * `amqp` → `rabbit` is not an identity. AMQP is a wire protocol and RabbitMQ is + * one implementation of it; a Qpid or ActiveMQ broker speaking AMQP is filed + * under `rabbit` here and the label is wrong about the product. It is mapped + * anyway because the alternative is a guaranteed MISS: Spring's own capture + * calls `@RabbitListener` `rabbit`, so an `amqp` document describing the very + * same queue would sit on a second node and the two would never meet. A label + * that is wrong about the vendor but right about the protocol family joins the + * pair; an honest `amqp` label splits it every time. + * + * The transport-security variants are that argument with the vendor doubt + * removed. AsyncAPI's SERVER vocabulary distinguishes `kafka` from + * `kafka-secure`; its BINDINGS vocabulary does not. Without these rows a + * secured cluster's own document contradicts itself, and with bindings absent + * it is worse and quieter: `kafka-secure
` never meets the + * `kafka
` Spring capture mints, and nothing reports the miss. TLS is + * a property of the connection, not of the place messages go. + */ + +/** + * A protocol name long enough to be a mistake. + * + * The broker is the THIRD string that reaches a graph identifier, alongside the + * address and the operation id, and it is the one that was left unbounded: + * `destinationNodeKey` is `` `${broker} ${address}` `` and `generateId` is + * `` `${label}:${name}` `` — concatenation both, no hashing. A one-megabyte + * protocol in a document that satisfies every other cap was measured producing + * a gigabyte of resident identifier strings, because the phase mints one id per + * node and one per edge. The longest name in AsyncAPI's vocabulary is + * `googlepubsub` at twelve characters, so this bound is generous by more than a + * factor of two and can only be reached on purpose. + */ +const MAX_PROTOCOL_LENGTH = 32; + +/** + * Spellings that differ between AsyncAPI's protocol vocabulary and the broker + * names this codebase already mints from source. + * + * Both AMQP versions collapse: `amqp1` is AMQP 1.0, a different wire format for + * the same family, and a service that documents one while its code speaks the + * other is describing one queue, not two. `mqtt5` collapses onto `mqtt` for the + * identical reason. + */ +const PROTOCOL_ALIASES: ReadonlyMap = new Map([ + ['amqp', 'rabbit'], + ['amqp1', 'rabbit'], + ['kafka-secure', 'kafka'], + ['secure-mqtt', 'mqtt'], + ['mqtts', 'mqtt'], + ['mqtt5', 'mqtt'], + ['wss', 'ws'], + ['stomps', 'stomp'], + ['https', 'http'], +]); + +/** + * AsyncAPI's binding vocabulary — the names a `bindings` map key may take. + * + * Closed on purpose; see the header. Adding a protocol here is a deliberate + * act, which is the point: the cost of a missing row is one document's + * destinations, and the cost of an open door is a node keyed on a vendor + * annotation that two unrelated services happen to share. + */ +const BINDING_PROTOCOLS: ReadonlySet = new Set([ + 'amqp', + 'amqp1', + 'anypointmq', + 'googlepubsub', + 'http', + 'https', + 'ibmmq', + 'jms', + 'kafka', + 'kafka-secure', + 'mercure', + 'mqtt', + 'mqtt5', + 'mqtts', + 'nats', + 'pulsar', + 'redis', + 'secure-mqtt', + 'sns', + 'solace', + 'sqs', + 'stomp', + 'stomps', + 'ws', + 'wss', +]); + +/** + * Protocols whose destinations this module refuses to mint, because the address + * alone is not the thing that identifies them. + * + * For a broker, the topic or queue name IS the namespace: two services naming + * `orders.v1` on Kafka are talking about one place, and dropping which cluster + * they used is a bounded, stated trade. For HTTP and WebSocket the HOST is the + * namespace and the address is only a path, so keying on the path alone makes + * every service that exposes `/events` — or `/health`, or `/api/v1/orders` — + * one node. That is unbounded, and it is a false join rather than a lost one. + * + * These are not lost information: an HTTP endpoint is a `Route`, which the + * routes phase already models with the method in its key. + */ +const NON_DESTINATION_PROTOCOLS: ReadonlySet = new Set(['http', 'ws']); + +/** + * Shape a protocol NAME must take, applied to both readers. + * + * The pass-through in {@link brokerForProtocol} is an argument about + * UNRECOGNIZED protocols — a name this codebase has not heard of is still the + * document's claim. It is not an argument about arbitrary text. A protocol name + * contains no whitespace, and one that did would collide in the node key, since + * `destinationNodeKey` joins broker and address with a space: `("kafka orders", + * "x")` and `("kafka", "orders x")` are then the same node. + * + * Learned twice. The check was added when that collision was first shown to be + * reachable, then dropped during a rewrite that moved the binding-key filtering + * into its own function — and the test written for the first lesson caught the + * second within the minute. + */ +function isProtocolToken(value: string): boolean { + return /^[a-z0-9][a-z0-9+._-]*$/.test(value); +} + +function normalize(protocol: string | undefined): string | undefined { + if (protocol === undefined) return undefined; + const trimmed = protocol.trim().toLowerCase(); + if (trimmed === '' || trimmed.length > MAX_PROTOCOL_LENGTH) return undefined; + if (!isProtocolToken(trimmed)) return undefined; + return trimmed; +} + +/** True when a broker names a transport whose addresses must not be keyed. */ +export function isNonDestinationBroker(broker: string): boolean { + return NON_DESTINATION_PROTOCOLS.has(broker); +} + +/** + * Normalize a `servers[].protocol` value to the broker half of a `Destination` + * key. Unrecognized protocols pass through; see the header. + * + * Returns `undefined` for a blank or implausibly long value — silence is not a + * claim, and a key built from an empty string would merge every silent + * document. + */ +export function brokerForProtocol(protocol: string | undefined): string | undefined { + const normalized = normalize(protocol); + if (normalized === undefined) return undefined; + return PROTOCOL_ALIASES.get(normalized) ?? normalized; +} + +/** + * Normalize a `bindings` MAP KEY to a broker, answering only for names in + * AsyncAPI's binding vocabulary. + * + * `$ref` and `x-` extensions live in this namespace legitimately, so anything + * unrecognized is silence rather than a broker. + */ +export function brokerForBindingKey(key: string | undefined): string | undefined { + const normalized = normalize(key); + if (normalized === undefined || !BINDING_PROTOCOLS.has(normalized)) return undefined; + return PROTOCOL_ALIASES.get(normalized) ?? normalized; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts b/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts index f02386192..f99fa433b 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts @@ -281,6 +281,31 @@ export interface SpringDestinationResolvers { * order is fixed now rather than renegotiated later. Nothing supplies it * today, so step 4 is a no-op and such destinations stay unresolved with the * reason the earlier step recorded. + * + * ── STILL UNSUPPLIED, AND NOW FOR A REASON RATHER THAN FOR WANT OF A READER ── + * + * `core/ingestion/asyncapi/document.ts` reads AsyncAPI 3.x documents, and + * `pipeline-phases/spring-destinations.ts` emits what they state as + * destinations of their own. It does NOT feed this hook, and the gap is a + * decision: + * + * A document names addresses; it does not name the method that uses one. To + * hand an address to THIS candidate, something has to choose which of the + * document's operations belongs to it. Partitioning by (broker, action) is + * the only division both sides agree on, and it is a weak one: a service with + * several listeners on one broker puts them all in one bucket. Any bucket + * holding more than one operation forces a heuristic, and a wrong heuristic + * puts a REAL address on a joining node under the wrong site — a false + * connection wearing the clothes of a resolved one, which is the exact + * outcome this module's keying rule exists to prevent. Only a bucket of size + * one is a fact rather than a guess. + * + * Two things would change that, and neither is a heuristic: a document whose + * operations carry the implementing symbol, or a configuration source that + * answers the `${key}` this candidate already recorded. The second is the + * stronger of the two — a key-to-value lookup is exact where a document match + * is a guess — and it wants its own resolver rather than this one, because + * what it needs is the placeholder key, not the candidate. */ readonly specification?: (candidate: SpringDestinationCandidate) => string | null; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts b/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts index 0106eaa98..47c672648 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts @@ -7,6 +7,15 @@ * inbound and outbound facts are captured during parse and survive the parse * cache; until now nothing read them. * + * Since `asyncApiSpecPath`, the phase has a SECOND source that is not Spring + * and not source code at all: AsyncAPI documents read off disk + * (`ingestion/asyncapi/document.ts`). They mint the same `Destination` nodes on + * the same key, so both sources meet on one node. The reader is deliberately + * framework-neutral and lives outside `frameworks/spring/`; only the emit is + * hosted here, because this is where the node's keying rule is enforced and + * splitting that rule across two phases is how it drifts. The phase name is + * accurate about its origin rather than its current contents. + * * Shaped after `Route` + `HANDLES_ROUTE` in `routes.ts` — a framework overlay * node keyed by what it names, with the callable pointing at it, down to the * detail that the key pairs the address with the one dimension that can make @@ -56,12 +65,16 @@ * prevent. * * @deps parse, scopeResolution, springConfig - * @reads Spring messaging capture facts, Method/Function nodes, Property nodes - * @writes Destination nodes; CONSUMES_FROM / PUBLISHES_TO / USES edges + * @reads Spring messaging capture facts, Method/Function nodes, Property nodes, + * AsyncAPI documents under `options.asyncApiSpecPath` (filesystem) + * @writes Destination nodes; synthetic File nodes for out-of-tree documents; + * CONSUMES_FROM / PUBLISHES_TO / USES edges */ +import path from 'node:path'; import type { GraphNode, Range } from 'gitnexus-shared'; import { generateId } from '../../../lib/utils.js'; +import { readAsyncApiDocuments } from '../asyncapi/document.js'; import { logger } from '../../logger.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import { SPRING_CONFIG_DESCRIPTION } from '../frameworks/spring/config-bindings.js'; @@ -101,6 +114,37 @@ export interface SpringDestinationsOutput { readonly refusalsByReason: Readonly>; /** Destination -> Property provenance edges for `${key}` placeholders. */ readonly configKeyLinks: number; + /** + * What reading AsyncAPI documents contributed, present only when + * `asyncApiSpecPath` was configured. Absent means "not asked for", which is + * deliberately distinguishable from a configured path that yielded nothing — + * a mistyped directory and a repository with no documents are different + * problems with different fixes, and one zero cannot say which happened. + */ + readonly specDocuments?: SpecDocumentStats; +} + +export interface SpecDocumentStats { + /** Entries skipped because they were symbolic links, and whether a bound + * stopped the walk. Both make every other number here a FLOOR, and a floor + * reported as a total is the failure this whole block exists to prevent. */ + readonly symlinksSkipped: number; + readonly truncated: boolean; + /** Files considered under the configured path. */ + readonly scanned: number; + /** Files that parsed as an AsyncAPI 3.x document and yielded an operation. */ + readonly accepted: number; + /** Operations normalized to a (broker, address, action) triple. */ + readonly operations: number; + /** Destination nodes this reading minted that no source site had already. */ + readonly destinations: number; + /** CONSUMES_FROM + PUBLISHES_TO edges from documents. */ + readonly edges: number; + /** Document- and operation-level refusals, by reason. Kept apart from + * `refusalsByReason` above: that number is the denominator of the SOURCE + * unresolved fraction, and a mistyped specification directory must not be + * able to make the source look worse than it is. */ + readonly refusalsByReason: Readonly>; } /** @@ -274,6 +318,192 @@ function edgeReason(candidate: SpringDestinationCandidate): string { return `spring-${candidate.source}:${element}${exchange}`; } +/** + * Pseudo-path prefix for a document that is not a file of this repository. + * + * An edge needs a source node, and the emit below refuses to attach one to a + * `File` that does not exist — so a document supplied from outside the working + * tree needs an identity minted for it. The same answer + * `frameworks/spring/actuator-runtime.ts` gives for Actuator snapshots + * (`spring-actuator:`): a prefixed pseudo-path that a real + * repo-relative path is not expected to take. + * + * That is a CONVENTION, not a guarantee, and the difference is worth stating + * because the neighbouring prefix states it too strongly. A colon is a legal + * POSIX filename character, so a committed file literally named + * `asyncapi:orders.yaml` would share this identity — costing one merged node + * and a misattributed edge, never a wrong address. Windows cannot express the + * collision at all. It is accepted on the same terms the Actuator prefix + * already is rather than escaped, because an escape would have to be applied to + * both prefixes at once to be worth anything. + */ +const DOCUMENT_FILE_PREFIX = 'asyncapi:'; + +/** + * The `File` node an AsyncAPI operation's edge hangs off. + * + * A document COMMITTED to the repository already has a real `File` node, and + * using it is strictly better: the edge lands on something the reader can open, + * and the ordinary per-file writeback keeps it honest. Only a document from + * outside the tree gets a synthetic node. + */ +function documentFileNodeId( + ctx: PipelineContext, + configuredRoot: string, + documentPath: string, +): string { + const repoRelative = path.relative(ctx.repoPath, documentPath); + if (repoRelative !== '' && !repoRelative.startsWith('..') && !path.isAbsolute(repoRelative)) { + const realId = generateId('File', repoRelative.split(path.sep).join('/')); + if (ctx.graph.getNode(realId) !== undefined) return realId; + } + // Relative to the CONFIGURED root, not to the filesystem root: an absolute + // path would put a machine's directory layout into the graph, and two + // machines indexing the same documents would then disagree about their ids. + const relative = path.relative(configuredRoot, documentPath); + const label = + relative === '' || relative.startsWith('..') + ? path.basename(documentPath) + : relative.split(path.sep).join('/'); + const filePath = `${DOCUMENT_FILE_PREFIX}${label}`; + const id = generateId('File', filePath); + if (ctx.graph.getNode(id) === undefined) { + ctx.graph.addNode({ + id, + label: 'File', + properties: { name: path.basename(documentPath), filePath }, + }); + } + return id; +} + +/** + * Mint destinations stated by AsyncAPI documents, with no claim about code. + * + * ── WHY THIS DOES NOT TRY TO FIND THE HANDLER ───────────────────────────── + * + * A document says an address is sent to or received from; it does not say by + * which method. Guessing that mapping is a real temptation and a bad trade: the + * addresses in one document partition by (broker, action) into buckets that + * usually hold more than one operation, so any assignment beyond a bucket of + * size one is a heuristic — and a wrong one silently attaches a real address to + * the wrong handler, which is a false connection dressed as a resolved one. + * + * So this claims only what the document actually states: that THIS SERVICE + * talks to that address on that broker in that direction. The edge therefore + * starts at the document, not at a callable. That is a weaker statement than a + * source-derived edge and it is worth having anyway, because it is available in + * cases where the source cannot supply one at all — a listener registered + * programmatically, a broker this codebase has no patterns for, or a language + * whose messaging idiom nobody has taught it yet. + * + * The node itself is the ordinary resolved `Destination`: same key, same + * `address` property, so a document and a source site that name one address on + * one broker land on ONE node and the two halves of a conversation meet. That + * is the whole point, and it is why this mints nothing of its own invention. + */ +async function emitSpecDestinations( + ctx: PipelineContext, + specPath: string, +): Promise { + const read = await readAsyncApiDocuments(ctx.repoPath, specPath); + const configuredRoot = path.resolve(ctx.repoPath, specPath); + let destinations = 0; + let edges = 0; + + for (const operation of read.operations) { + const nodeId = generateId( + 'Destination', + destinationNodeKey(operation.broker, operation.address), + ); + // Runs AFTER the source pass, so a site that resolved this address already + // owns the node and keeps its own `resolution` provenance. First writer + // wins and the order is fixed, so the property is deterministic rather than + // a race — and `literal` is the more informative of the two answers anyway. + if (ctx.graph.getNode(nodeId) === undefined) { + ctx.graph.addNode({ + id: nodeId, + label: 'Destination', + properties: { + name: operation.address, + // Empty for the same reason every connecting destination carries it + // empty: the node is shared, and stamping it with the document's path + // would make it collateral damage of that path's next writeback. + filePath: '', + address: operation.address, + // NOT `'specification'`, though the address did come from one. That + // value belongs to `SpringDestinationVia` and means "a CODE + // CANDIDATE was resolved through the step-4 resolver hook" — a + // different fact with a code site behind it. Reusing it would make a + // query that groups destinations by provenance unable to separate an + // address a document merely states from one a document was used to + // resolve, and the second of those is a claim about source that this + // node is not making. + resolution: 'asyncapi-document', + broker: operation.broker, + }, + }); + destinations += 1; + } + + const sourceId = documentFileNodeId(ctx, configuredRoot, operation.documentPath); + const type = operation.action === 'receive' ? 'CONSUMES_FROM' : 'PUBLISHES_TO'; + const reason = `asyncapi:${operation.operationId}`; + ctx.graph.addRelationship({ + id: generateId(type, `${sourceId}->${nodeId}:${reason}`), + sourceId, + targetId: nodeId, + type, + confidence: 1.0, + reason, + }); + edges += 1; + } + + const stats: SpecDocumentStats = { + symlinksSkipped: read.symlinksSkipped, + truncated: read.truncated, + scanned: read.documentsScanned, + accepted: read.documentsAccepted, + operations: read.operations.length, + destinations, + edges, + refusalsByReason: read.refusals as Readonly>, + }; + + // Unconditional, and not `isDev`-gated like the summary below it. The tally + // above is justified on the grounds that an operator must be able to tell a + // mistyped directory from a repository with no documents — and that + // justification is only true if the operator can SEE it. A configured path + // that produced nothing is the one outcome where silence and success look + // identical from outside, which is why `spring-auto-configuration.ts` warns + // unconditionally for the same class of input. + if (stats.accepted === 0 || stats.truncated) { + // Repo-relative when the path is inside the repository, bare name when it + // is not. The same change refuses to persist this path to index metadata on + // the grounds that it would record an operator's directory layout; applying + // that reasoning to metadata and not to logs would be holding one rule in + // two places. + const resolved = path.resolve(ctx.repoPath, specPath); + const relative = path.relative(ctx.repoPath, resolved); + const reportedPath = + relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative) + ? relative.split(path.sep).join('/') + : path.basename(resolved); + logger.warn( + { + asyncApiSpecPath: reportedPath, + ...stats, + }, + stats.accepted === 0 + ? '⚠️ No AsyncAPI document under the configured path yielded a destination.' + : '⚠️ AsyncAPI document reading hit a bound; the destinations below are a floor, not a total.', + ); + } + + return stats; +} + export const springDestinationsPhase: PipelinePhase = { name: 'springDestinations', // `parse` supplies the file list and the harvested constants; `scopeResolution` @@ -362,13 +592,28 @@ export const springDestinationsPhase: PipelinePhase = ); } } + // Documents are read whether or not the source pass found anything, and + // that is the point of the closure rather than a straight call here: a + // repository whose messaging is invisible to the source patterns — a broker + // with no rules, a listener registered programmatically — is exactly the + // case a published document exists to cover, and an early return keyed on + // source sites would skip the documents precisely there. + // + // Always invoked AFTER the source emit, so a site that resolved an address + // owns its node first and keeps its own provenance. + const specPath = ctx.options?.asyncApiSpecPath; + const readSpecifications = async (): Promise => + specPath === undefined ? undefined : emitSpecDestinations(ctx, specPath); + if (sites.length === 0) { + const specDocuments = await readSpecifications(); return { resolvedDestinations: 0, unresolvedDestinations: 0, edges: 0, refusalsByReason, configKeyLinks: 0, + ...(specDocuments === undefined ? {} : { specDocuments }), }; } @@ -548,9 +793,20 @@ export const springDestinationsPhase: PipelinePhase = edges += 1; } + const specDocuments = await readSpecifications(); + if (isDev) { + const fromSpec = + specDocuments === undefined + ? '' + : `, +${specDocuments.destinations} from ${specDocuments.accepted} document(s)`; + // The breakdown, not just the totals. The unresolved FRACTION is the + // number this feature is judged on, and a bare count of unresolved + // destinations says how big the gap is without saying what would close + // it — which is the only question an operator can act on. logger.info( - `📮 Spring destinations: ${resolvedDestinations} resolved, ${unresolvedDestinations} unresolved, ${edges} edges`, + { refusalsByReason, ...(specDocuments === undefined ? {} : { specDocuments }) }, + `📮 Spring destinations: ${resolvedDestinations} resolved, ${unresolvedDestinations} unresolved, ${edges} edges${fromSpec}`, ); } @@ -560,6 +816,7 @@ export const springDestinationsPhase: PipelinePhase = edges, refusalsByReason, configKeyLinks, + ...(specDocuments === undefined ? {} : { specDocuments }), }; }, }; diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 00892d07c..ac1b120e8 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -74,6 +74,20 @@ export interface PipelineOptions { springActuatorPath?: string; /** Repo-relative Actuator inputs retained only for a cleanup scan. */ springActuatorScanExclusions?: readonly string[]; + /** + * Explicit local AsyncAPI 3.x document input, read by the `springDestinations` + * phase. Accepts a directory of documents or a single document; the path is + * resolved against the repository root, so a committed `docs/asyncapi` and an + * absolute cache populated out of band are equally natural. Undefined keeps + * specification reading completely disabled. + * + * There is deliberately no glob-based auto-discovery to go with it. Scanning + * a repository for anything that parses as a document would make every + * existing index grow destination nodes on its next run without an operator + * having decided anything — the same reason new contract extractors ship + * opt-in rather than on. + */ + asyncApiSpecPath?: string; /** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */ springAopMaxCandidateInspectionsPerAdvice?: number; /** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */ diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index be591792e..2dfd80b93 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -469,6 +469,11 @@ export interface AnalyzeOptions { * the Spring enrichment phase. Undefined keeps static-only analysis. */ springActuatorPath?: string; + /** + * Explicit local AsyncAPI 3.x document input, forwarded to the destination + * phase. Undefined keeps source-only address resolution. + */ + asyncApiSpecPath?: string; /** * The caller will `process.exit()` immediately after this analyze returns (the * CLI `analyze` command). When set, the finalize/error close CHECKPOINTs for @@ -1708,6 +1713,35 @@ async function runFullAnalysisInner( const springActuatorScanExclusions = retainedActuatorInputs.length === 0 ? undefined : retainedActuatorInputs; + // AsyncAPI documents are the same class of input as Actuator snapshots and + // need the same treatment, for a reason git cannot see: the documents live + // outside the tree as often as in it, and NOTHING about replacing one moves + // the commit or dirties the working tree. Without this, the second run of an + // out-of-band cache — the workflow the option exists for — takes the + // already-up-to-date fast path below, never opens a document, and serves the + // previous run's addresses while reporting success. Measured, not reasoned: + // editing a document and re-running printed "Already up to date" and left the + // old address in the graph. + // + // Forcing the rebuild also settles a second defect for free. A synthetic + // `File` node for an out-of-tree document (`asyncapi: